{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Annealing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Installing packages:\n",
      "\t.package(path: \"/home/jupyter/notebooks/swift/FastaiNotebook_04_callbacks\")\n",
      "\t\tFastaiNotebook_04_callbacks\n",
      "With SwiftPM flags: []\n",
      "Working in: /tmp/tmpmx6k2jyo/swift-install\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n",
      "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)[1/2] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "[2/3] Merging module jupyterInstalledPackages\n",
      "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n",
      "Initializing Swift...\n",
      "Installation complete!\n"
     ]
    }
   ],
   "source": [
    "%install-location $cwd/swift-install\n",
    "%install '.package(path: \"$cwd/FastaiNotebook_04_callbacks\")' FastaiNotebook_04_callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "//export\n",
    "import Path\n",
    "import TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import FastaiNotebook_04_callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('inline', 'module://ipykernel.pylab.backend_inline')\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%include \"EnableIPythonDisplay.swift\"\n",
    "IPythonDisplay.shell.enable_matplotlib(\"inline\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "let data = mnistDataBunch(flat: true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "let (n,m) = (60000,784)\n",
    "let c = 10\n",
    "let nHid = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "func optFunc(_ model: BasicModel) -> SGD<BasicModel> {return SGD(for: model, learningRate: 1e-2)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "func modelInit() -> BasicModel {return BasicModel(nIn: m, nHid: nHid, nOut: c)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeAvgMetric(metrics: [accuracy]),\n",
    "                     learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "//Crashes! See: SR-10436\n",
    "//learner.delegates = [type(of: learner).TrainEvalDelegate(), type(of: learner).AvgMetric(metrics: [accuracy])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: [0.3047825, 0.9138]\n",
      "Epoch 1: [0.24763964, 0.9286]\n"
     ]
    }
   ],
   "source": [
    "learner.fit(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Recorder's role is to keep track of the loss and our scheduled learning rate. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "import Python\n",
    "public let np = Python.import(\"numpy\")\n",
    "public let plt = Python.import(\"matplotlib.pyplot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "public func plot<S1, S2>(_ arr1: [S1], _ arr2: [S2], logScale:Bool = false, xLabel: String=\"\", yLabel: String = \"\") \n",
    "    where S1:PythonConvertible, S2:PythonConvertible{\n",
    "    plt.figure(figsize: [6,4])\n",
    "    let (npArr1, npArr2) = (np.array(arr1), np.array(arr2))\n",
    "    if logScale {plt.xscale(\"log\")} \n",
    "    if !xLabel.isEmpty {plt.xlabel(xLabel)}\n",
    "    if !yLabel.isEmpty {plt.ylabel(yLabel)}    \n",
    "    let fig = plt.plot(npArr1, npArr2)\n",
    "    plt.show(fig)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "extension Learner where Opt.Scalar: PythonConvertible{\n",
    "    public class Recorder: Delegate {\n",
    "        public var losses: [Loss] = []\n",
    "        public var lrs: [Opt.Scalar] = []\n",
    "        \n",
    "        public override func batchDidFinish(learner: Learner) {\n",
    "            if learner.inTrain {\n",
    "                losses.append(learner.currentLoss)\n",
    "                lrs.append(learner.opt.learningRate)\n",
    "            }\n",
    "        }\n",
    "        \n",
    "        public func plotLosses(){\n",
    "            plot(Array(0..<losses.count), losses.map{$0.scalar}, xLabel:\"iteration\", yLabel:\"loss\")\n",
    "        }\n",
    "        \n",
    "        public func plotLRs(){\n",
    "            plot(Array(0..<lrs.count), lrs, xLabel:\"iteration\", yLabel:\"lr\")\n",
    "        }\n",
    "        \n",
    "        public func plotLRFinder(){\n",
    "            plot(lrs, losses.map{$0.scalar}, logScale: true, xLabel:\"lr\", yLabel:\"loss\")\n",
    "        }\n",
    "        \n",
    "    }\n",
    "    \n",
    "    public func makeRecorder() -> Recorder {\n",
    "        return Recorder()\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Utility optional property to get backour `Recorder` if it was created by a utility function. This doesn't always work properly for unkwnon reasons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "//TODO: Fix\n",
    "extension Learner where Opt.Scalar: PythonConvertible{\n",
    "    public var recorder: Learner.Recorder? {\n",
    "        for callback in learner.delegates {\n",
    "            if let recorder = callback as? Learner.Recorder { return recorder }\n",
    "        }\n",
    "        return nil\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeAvgMetric(metrics: [accuracy]), \n",
    "                     learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std), learner.makeRecorder()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: [0.29703593, 0.9139]\n",
      "Epoch 1: [0.24783066, 0.9272]\n"
     ]
    }
   ],
   "source": [
    "learner.fit(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd5wTdfoH8M+TbKH3pUhbUBDpIAgoKAoqYMF2tju7cna987wfnOXs/Twbp+LpWQ7RU7DSmwgiZUF6XXpnabsLW7P7/f0xM9lJMmm7mSTLfN6vFy+SySR5kt39PvPtopQCERE5lyvRARARUWIxERARORwTARGRwzEREBE5HBMBEZHDpSQ6gGg1adJEZWZmJjoMIqJqZdmyZYeUUhlWj1W7RJCZmYmsrKxEh0FEVK2IyI5gj7FpiIjI4ZgIiIgcjomAiMjhmAiIiByOiYCIyOGYCIiIHI6JgIjI4WxLBCLSWkTmisg6EVkrIg9ZnDNYRHJFZIX+70m74gGAGWv342BekZ1vQURU7dg5ocwD4BGl1HIRqQtgmYjMVEqt8ztvvlLqUhvj0IIpK8eoz5ahfUZtzHlksN1vR0RUbdhWI1BK7VNKLddv5wNYD6ClXe8Xjqdc24Bn15GCRIVARJSU4tJHICKZAHoBWGzx8AARWSkiU0WkS5DnjxKRLBHJysnJqVQM5fpObAKp1POJiE5WticCEakDYCKAh5VSeX4PLwfQVinVA8DbAL61eg2l1DilVB+lVJ+MDMs1k8Iq02sEwjxAROTD1kQgIqnQksB4pdQk/8eVUnlKqeP67SkAUkWkiR2xMBEQEVmzc9SQAPgQwHql1OtBzmmunwcROUuP57Ad8Rh9BGwaIiLyZeeooXMA3ARgtYis0I/9DUAbAFBKvQfgGgD3iIgHQCGA65XSG/NjrFxPBC7mASIiH7YlAqXUAiD05bdS6h0A79gVg9mU1fsAACdKyuLxdkRE1YZjZhaX21LPICKq/hyTCNJSHPNRiYii4pjSMZ2JgIjIkmNKx5YNayY6BCKipOSYRHD2qdr0hE7N6yY4EiKi5OKYRGDYsD8/0SEQESUVxyUCIiLyxURARORwTARERA7HREBE5HBMBEREDueoRPCH/m3QuHZaosMgIkoqjkoEKS6XdzlqIiLSOCoRuF3i3aCGiIg0jkoEKS5BaVl5osMgIkoqjkoErBEQEQVyVCJIcWt9BDZtgkZEVC05KxHo+1SyUkBEVMFRicCQV1ia6BCIiJKGoxKBsW/xK9M3JDgSIqLk4ahEYHQU57JGQETk5ahE4Nb7CKas3p/gSIiIkoejEkGKWxIdAhFR0nFUIkh1O+rjEhFFxFElo0tYIyAi8ueoRHBFz1MSHQIRUdJJSXQA8fSH/m0xefU+HCvgqCEiIoOjagQigmb1aqCwtCzRoRARJQ1HJQIAqJnqRmEJEwERkcF5iSDNzRoBEZGJ8xJBqhtFTARERF62JQIRaS0ic0VknYisFZGHLM4REXlLRLJFZJWI9LYrHkPNVDdKyxQ3qCEi0tlZI/AAeEQp1RlAfwD3iUhnv3OGA+ig/xsF4F0b4wGgNQ0BYPMQEZHOtkSglNqnlFqu384HsB5AS7/TRgL4VGkWAWggIi3sigkAaqTqiYAdxkREAOLURyAimQB6AVjs91BLALtM93cjMFlAREaJSJaIZOXk5FQpliZ10gAAOfnFVXodIqKThe2JQETqAJgI4GGlVF5lXkMpNU4p1Ucp1ScjI6NK8TStVwMAEwERkcHWRCAiqdCSwHil1CSLU/YAaG2630o/ZpsaKVrTULGHncVERIC9o4YEwIcA1iulXg9y2vcAbtZHD/UHkKuU2mdXTACQlqJ95BenrrfzbYiIqg071xo6B8BNAFaLyAr92N8AtAEApdR7AKYAGAEgG0ABgNtsjAcAkK4ngh2HC+x+KyKiasG2RKCUWgAg5LrPSikF4D67YrCSnuq4OXRERCE5rlSsleaoBVeJiMJyXCKok16RCLbkHE9gJEREycFxicDs/XlbEh0CEVHCOToReMpUokMgIko4RyeC0F3ZRETO4OhEIMwEREQOTwTMA0REzk4E5Yp9BEREjk4Ek5bbuqwREVG14OhEAACKtQIicjhHJoLGtdO8t48XexIYCRFR4jkyEUx9aJD3dgmXoyYih3NkIjA2pwEATzmbhojI2RyZCADg5au7AQBKy1gjICJnc2wiSHVrH53LTBCR0zk2EaQYiYBNQ0TkcI5NBKkubVqxp5xNQ0TkbI5NBG4jEbBpiIgczrGJwOgjyCssTXAkRESJ5dhE0KphTQDAjiPcxJ6InM2xicCYS1BQUpbgSIiIEsuxiaBmqhsA8OyP6/Dsj+sSHA0RUeI4NhGkuis2I/hwwbYERkJElFiOTQTCXWmIiAA4OBEQEZGGiYCIyOGYCIiIHI6JgIjI4RydCFo2qJnoEIiIEs7RieDft/RJdAhERAnn6ETQsFZa+JOIiE5ytiUCEflIRA6KyJogjw8WkVwRWaH/e9KuWIJJT3F0HiQiAgCk2PjaHwN4B8CnIc6Zr5S61MYYQkpPZSIgIrKtJFRK/QzgiF2vHwtpbiYCIqJEl4QDRGSliEwVkS7xfvMUJgIiooQmguUA2iqlegB4G8C3wU4UkVEikiUiWTk5OTENol+7RgCAr7J2xfR1iYiqi4QlAqVUnlLquH57CoBUEWkS5NxxSqk+Sqk+GRkZMY2jXs1UAMBni3bE9HWJiKqLhCUCEWku+hKgInKWHsvhRMVzotiTqLcmIkoo20YNicgEAIMBNBGR3QD+DiAVAJRS7wG4BsA9IuIBUAjgeqVU3HeSNxajLivnJvZE5Ey2JQKl1A1hHn8H2vDShHLp+xKUljEREJEzOX7YjLE/jae8PLGBEBElCBOBkQhYIyAih2Ii0HsJDp8oSXAkRESJ4fhE4HJx72IicjbHJ4InLjkj0SEQESVURIlARB4SkXqi+VBElovIRXYHFw9N69Xw3n5p6gYc53wCInKYSGsEtyul8gBcBKAhgJsAvGRbVAny3rwteGPmpkSHQUQUV5EmAqMhfQSAz5RSa03HTionSsoSHQIRUVxFmgiWicgMaIlguojUBXDSDLwfdW577+3SspPmYxERRSTSmcV3AOgJYKtSqkBEGgG4zb6w4qtJnYotKz1MBETkMJHWCAYA2KiUOiYifwDwOIBc+8KKL/PyEt+u2JvASIiI4i/SRPAugAIR6QHgEQBbEHoLymrFf627BKx9R0SUMJEmAo++MuhIAO8opcYCqGtfWPHlX+4XsMOYiBwk0kSQLyJjoA0bnSwiLuhLSp8M/K//j3C5CSJykEgTwXUAiqHNJ9gPoBWAV22LKs78awRMBETkJBElAr3wHw+gvohcCqBIKXXy9BH41Qnyizi7mIicI9IlJq4FsATA7wBcC2CxiFxjZ2Dx5F8j4N4EROQkkc4jeAxAX6XUQQAQkQwAswB8bVdg8eQ/SojbVhKRk0TaR+AykoDucBTPTXr+xT63rSQiJ4m0RjBNRKYDmKDfvw7AFHtCir8aqW6f+6wREJGTRJQIlFKPisjVAM7RD41TSn1jX1jxVbeG79fAPgIicpKIm3eUUhOVUn/W/500SQAALut+is/9w8c5fJSInCNkIhCRfBHJs/iXLyJ58QrSbg1rp6F7q/re+8/8uC6B0RARxVfIpiGl1EmzjEQ4J+XmCkREEThpRv5UlQhTARE5ExOBzmXKA83qpScuECKiOGMi0JlrBAfyijFz3QHM3XAQf/wsCzn5xVix61gCoyMisk+k8whOej1bN8CyHUe991fuOoZ35mYDAJbvPIac/GJsf+mSRIVHRGQb1gh0o4d3wo8PDPTe/2XLIe/tnPziRIRERBQXTAS6VLcLXVtWDCH9bSebgojIGZgIiIgcjomAiMjhbEsEIvKRiBwUkTVBHhcReUtEskVklYj0tiuWWOGm9kR0MrKzRvAxgGEhHh8OoIP+bxSAd22MJSa4KCkRnYxsSwRKqZ8BHAlxykgAnyrNIgANRKSFXfHEQjlrBER0EkpkH0FLALtM93frxwKIyCgRyRKRrJycnLgEZ4X7FBDRyahadBYrpcYppfoopfpkZGQkLA7WCIjoZJTIRLAHQGvT/Vb6saTFGgERnYwSmQi+B3CzPnqoP4BcpdS+BMYTFjcuI6KTkW1rDYnIBACDATQRkd0A/g4gFQCUUu9B2/N4BIBsAAUAbrMrllgp05uGdhw+gbxCD7qZNrMhIqqubEsESqkbwjyuANxn1/vbwWgaOu/VnwCAi9AR0UmhWnQWx9OA9o0BAN1aBl7t788twv+W7go4TkRUnXEZaj+f3nEWSsvK8facbKzek+vz2C3/WYIjJyo2tn91+gY8enGneIdIRBRTrBH4SXW7UCstBZ6ywJ5hcxIAgLFzt8QrLCIi2zARBOHhUFEicggmgiA8ZUwEROQMTARBsEZARE7BRBCEVR8BEdHJiIkgiGiXk1BKYcKSnThR7LEpIiIiezARBHFlb8uFUINavvMoxkxajce/tdyHh4goaTERBDGoQwa2v3QJvr57QITPEADAtkMnvEc++3U7Rrw5n81MRJTUmAjC6JPZKOTjU1fvw7GCEtROdwMA8otKvY89N3k91u3Lw3E2FxFREmMiqKJ7xi/HAxN+w/EirbDPL6oo9Is9Wk2AA5CIKJkxEcTA3mOFuOa9XwH4JgID9zEgomTGRBADW3Iq+gWKPGX4w78XY8HmQ95j3NmMiJIZF52LMaWABdmHsGZvxYJ1rBEQUTJjjSACzevViPo5YrpdVq7NMZi78WDI55z57Ey8MGV91O+VzJRSUKwRESU1JoIILPrbkCiGkQYqVwpjJq3Gbf9ZGvK8wydKMO7nrZV+n2TUbswUPP3DukSHQUQhMBFEKMUd3VclUlEnMLcM5RaUInP0ZHy0YFusQktaRk3g44XbExsIEYXERBAhl4Q/JxhzH8HuYwUAgP9lBd/p7LXpG7F0+xHLx/KLSmM6QW3p9iPILSgNf2IlsEWIqHpgIohQl1Oi26i+sKTMe9s8auiStxYA0DbACeadudn4nT4c1V+3p2bgka9WRhWLIb+oFOWmpJRbWIrfvfcrHv7yt0q9XjgcLUVUPTARRMjtErx8dbeIzy8srUgEVqOG3H5VjGiuyr9bsTficw0nij3o9tQMvDRtA3YeLsDF//wZa/StOHcfLYz69SLBwVJE1QMTQRSu7dMafx12etTPs0oEqW7fRHDFv34J+zpVGX1jTHT7bsUefDB/KzYeyMd3K/bosdjzaxCLGsGSbUeQOXqyN2kRUexxHkEURASnN6sb9fP25RYFHEtxaYVv72dnoker+j6L1Vk5VlCCZ3503uibmev2AwAWbjmEri2ja54josiwRhAHd32aFXAsRa8RHDlRgrkbc8K+xhuzNmPS8j1VjsV8kW53E34sagTG6Cs2MxHZh4kggaIZ/eNfqC7MPoRzX5nr0ykdyomSwDWQKlO4Ltl2BGc8MQ3HCkrCnmv1+oUlZXhnzuaIP7sxCpcdz0T2YSKIUqzKo/mbD2HQK3Mr/b5jvlmNnUcKsOdYZB29N36wCEBFwQoACtF/mHfmZqOwtAy/7TqG7IPHMXdD8NnSVoX3W3M247UZm/D1st0RvZ/oc7SZB4jsw0RQSV1b1qvya1j1HZgdOl7sve1faB89oV2R10iN7Ed4IE97LZ8CVfn8Z2nn4QKc+exMfLZoBzJHT8bmA/kAtCU0hr4+D7d9HHy2tLK46C/Q92YoKo2sJmMMruIyFdqmR5mjJ2OT/jMgihUmgkpySRVmmEXoCX3by9s/Xor/Ltrp85hRLlalfIykueWrZbtw+ESJNxYjeU1bs79Krx9p2K4Y9BGMnrgKE5bsDH+iDZRS+OfMTdhxOPRggEhMWb0PAPDNb1XvK0oUpZTP5k3+vluxB0dOhG92pNhiIoiSUR4ZnZj+w0BjqcRTjh2HT2CORfOLEYdVgXwgrwjvz9uCFbuOhXz9UGXr8WIPtuYcD/r4F0uDz4w2WCUCiTKBxqKP4IuluzBm0upKP78qdh8txJuzN+POTwIHDFRWda4cfbxwO7o9NQO7jxYEPLYvtxAPfbECvZ+dadn3dfh4sc+ESIodJoJKMsp/lwhuHtDWlvcoVwoTg4wUMppKnp+yHvM35/g0nfR7YTZenLoBV4y1nptQUbj6vpbZLR8twQX/mFeF6EMnmkgLMyNxRFv4JXLV0/yiUrw+cxM8ZeUo0TvFPTEowOyuhL41ezN+XBX9ZMVoTF+rXbjsPByYCEo9Fd/R6EmrfB47dLwYZz43C/+ctSkmcWQfPI5S7iXuxUQQJaPN2piEZWcT0dyNOXhv3hbLx8zlyk0fLomoqSbwNYIXTst2HI369aJ5/cibhvTzLV7r8PHioJ/7lekb0W7MlLCjk8rLVUyaIpZsO4K3Zm8GoK0V9dbszfhh1V7vFWxl1qoqKi3D+/O2xG0/i9dnbsL9n9uz3EgwSils2J8HwDfRZR/0rY3m5Gt9XDPWHqjye+49Voihr8/D85MTv+R71vYj6PH0DNvW+4qUrYlARIaJyEYRyRaR0RaP3yoiOSKyQv93p53xxMJ5HTNw+znt8PyV2nIT5qUiBnVogqcv7+K937h2WpXfr8RjXZD5dx4fzC9GYUkZdh3xvdL69rc9OOelOd77uYWlyC0sNV4krMpeVOcWllpu22n8sZeVl0d0xW4kWqszb/8kC3f/d1nF5zH5UF/dNdyV+NtzstH72Zk4mB+8477YUxY21mvf/xWvz9SuVo3lRUo85d73NyYQRuOt2Zvx4tQNmLQ8shFW1dHHC7dj2BvzsWSb7yKLwa6vYnHddVQf+rx4m/XCjpGKxeKPb8/JRm5hKZbvqvqFV1XYlghExA1gLIDhADoDuEFEOluc+qVSqqf+7992xRMrKW4XnrysMzLqpAPQfjGN380hnZrilrMzvecue+JC2+LwL9/cLsE945cFDEn968RVPkNMiz3l3rWK9uVWHJ+wZCeW7wz8ZYykZmBVSPZ4egaGhGhaemHKhoj2KTC+W6vaxXZ9NrbV+wcruEs85T7J1dgsKNh6S7uPFuD0x6dF1Cfi/75KVSwv4r+2VLDnmp9vJFL/EVbBaonxtnp3LhZmHwp/oonA93tYrS8d4t+Z7n+eHYzvev2+POQVlUZV81q9OxenPTYVP4XZbKq6sLNGcBaAbKXUVqVUCYAvAIy08f3iyrgiNzcNxbNF2r+mkOoW/GQxQzlU+bN8Z0Vn8phJq3HVvxYit7DUZ7mLX7ceDhvLqt25eHX6hqjb5I19CoyOwdW7cwOGRrpcwUcNGcnhq6zdKPCbMGf8Uc9c59uUcPW7C3HOyxU1pNrpbgDaonz+SsvKsTBb+/yTV+2L6DN5ypXPSCcjjpQIBhVc+/6v6Pj4VJSWlaOsXAXU+uJROIYzfvEOrN6tFd6XvbMAN/57cZVez/yZzFf7q/fkYujrVeujiuQ9l+88iuFvzkf3p2bg2SiWcFmiLxNv9TcXjWTp+rYzEbQEYL6M2q0f83e1iKwSka9FpLXVC4nIKBHJEpGsnJyqffGxYpR56SmugA7N5U9ciCWPDYlrPMGaHqJtX+7x9Ayc/9pPUT1n5NhfMHbuFrQbMwWfLdoRVUJYtuMIznhyGn7aeBCXvbMAF/3zZ5/HxdtHoP3/jxkbvSOAjGPPT1mPl6Zu8Hme8bEfmODb5r16T663vRkAaqRoiaCotCKx7s8twsIth9Dhsan460TfTkt/czccROboyd77JZ7yipihvE1DkfQlLd1+FKVlCh0em4qbPwosYP0Tg6esHAu3RHdF7u9fP2Xjpg9DF+aesopa1GPfrMFl7yyo0ntGyr+fwA57TbXlyg7L/XlTTqUHJhjPS3SKT3Rn8Q8AMpVS3QHMBPCJ1UlKqXFKqT5KqT4ZGRlxDTCYBrVS8dCQDvj8rv5oUV/b07hxHa1PoFHtNDStqx27ro9lbou5b1dY/xKXlsX3muOFyeuDxgIEXtVe/a6278LCLdY1j4qZxdrneHtONiYs2Yn/Ze3CcdNV/LFKdrb5l89FpWXo/+Js3PhBZFe6n/vNT9ASgfaia/fm4ep3FwIAUqLsLf4l2/f7yC0oxYZ9vrWlN2Ztxo0fLA66iVEo6/fl4etlu/HKtI2Yvzl0Mhnx1nx0fHxqyOHEhgcm/IY5G8J36Eb7WxnLAWDmiwu3uUZfiTf5fMlO3PzREny/0t7RVnazc/XRPQDMpWAr/ZiXUsr82/5vAK/YGE9MiQj+dGFHAEC7JrXRulEtDO/aPOC8l6/pji9D7EYWK+H+mOOlTCls3B+8wPjoF+stOs17Nc/dcBDnd2oKAN7Cp6i0DJ8vrih0//q175V6ekpsrmkKgqzdFGknZbGn3JvqzPGG6yN436LdP6+wItH1eGZGwOPGFbO5hhOp4W/Oj/jcTQe094lkOPEPK/fih5V7sf2lSywfNzc1FpR4MFHvCA9XBFdmOZRg3py12fuaLlfVmnaNmpLRx6SUwkvTNuCa3q3QwWKl4ryiUtRJS/F532RgZ41gKYAOItJORNIAXA/ge/MJItLCdPdyAIkfz1UJbpdgRLcWUU+WShYb9sduyYKycmXZL/FV1i7sD7OkhuFAnnbejLX78ZW+JtEnv+7A374JPiksPcKlNgyXvDXf5wpw7saDKCwpg6fceiRIsETrfxGpoCyTRrhE8KJf0xYA7NBHgD3x3VrL5/yid9QmywSzaK+ojQJZe3Jkn8P8N/btb3u8vyuA1h+0OII+rWlrK4Ycm2sE+UUeZI6e7FPTNDtWUIIVu455l3cxMz77xgP5eH/eVvzBorltyup96P7UDLw2Y2PQz7Vo62H8ZjFow262JQKllAfA/QCmQyvg/6eUWisiz4jI5fppD4rIWhFZCeBBALfaFQ/FR1m5slwI79GvV6H/i7Mjeg2jPAi3R4NZJMMzzWs3rd2b59N/8vninXjiuzUhd2uzHsrrW3qVK+tOXatEsGZPLjJHT/Z2vga+dOiSMT9IgVVervCPGRsx7I2fMXfDQXjKyjFm0qqYLHNhsJr5G+10B/8VcaOZPX682IOHv1zh7d9QSuGuT7Nw3bhFQfsWDuQV4YUpFdeaSln/XPZa/P7OWncAPZ+ZiSvG/oJr3/81IOkZd4e9MV9/r8Ba2r3jlwMAfggxae/6cYtw5b8WBn3cLrb2ESilpiilOiqlTlVKPa8fe1Ip9b1+e4xSqotSqodS6nylVOBl0Umme6uTf3OVymylaWb8UdVMc0f8HPNVuDFByV+f52b53Nc6ciue+PWy3bgqxB+h0Tzx3Yo9yBw9OWCkEqAVSFZLe1j1EczQr0xnB2lTPxhhk49/s8m787bg7TnZ2LA/H7d9vBQrdx/DhCW78KcvV0T0epGYsS5wIp+5NvXl0p34NUi/D2Cd44LlAaUUXp/hO6O4TO/7Mgpc89Dn16YHXnFnH8xHvxdm+zRBArBsorGquy0y1TQ2WySaaJKgUtpGS8WeyBZejIdEdxY7wrNXdPXevnfwaQmMpHrILSyFp6y80ltoGldl4US7xIBRzhlNGnuPFQU2DamKsfFmIoJ5m3Lw0YKKPhJjRJG5L8As3Oq05vc0869h/KYPE452FnyJpzzo2j7m2pRRCJtb1f5v4mrcoC99How57sLSMlzznnUS3nOsELP19bbW78tDaVk5dulrFRkfyTx50Ti2ctcxb9PRS1MDk0M0FZhY9mHsPlqIGz9YjMe+WeM9dstHS4LOLs4tLA0YBh1rTARxcFP/thh6RjMAgZ2OV/ayGlHrbC9P24Axk1ZH1fZdmXbyaIfWPvbtahw5UeJtTrDqTwgWx5wNB3HLR0t8ths13j9YB3ooo4MMa73xg0U+beAA8Jy+lEKkHZR5RaUo9pSh4+NT8chXKy3PMc+7uOuTLNz4wSLvvJDK+Pv3a3HouPVSH/59b7PXH8Clb/sOYbX63keO/QUX6EOhrZqdlFIRL2Ln/3T/2eyVWQXkF7/JeBuDLC9+/+fLcdenWRH3sVUGE0HcWI8X7t22oc/9Hq0bxCme5PbVst0oi7J0Ly9XUS3HEO3Q2knL9+CFKeu9iWDYG/MD+kOCNfNYed+vmSIa5pnO5k8RbBguoHWMnij2WO4F8fi3FR3x3Z+a4W1GCza23tyBffhECRZuOYyXpwW27C7ccgh3frIUnrJynz6aSO06UhAw+GDNnoqmP2PYsLmgn2paf+qE3pdhlfQVwi9BMnZuNu7+bFnAFf/bc7J97ufkFwX0TYS70PCvkT7zo/WgAKOv7M5Pl0a8oVO0uHl9nPlf3ZjvzfrzufhpYw5Whlk+2il2Rtm5+b+sXRgdxXLTkW7zaVbiKfeZJew/4iqSZTNy8ovRoFZq1O8dTDSjdbr8fbrlcf/9LqzWiaoMYz5GXpEHB00dqMWeMoxfHH6PiJFjf8GUBwf5HLMqDP3LXP9kF6wjOlhh/casTWhWrwZetehvsDJhyS5MWOI7TLy0rBxuV/B+rkPHS5C1vaJvw5zgij1lKCotx/7ciubHNXvysCWCuRyVwUQQJ8H+Vnu2boB/XtcDLRvUwmlN6+LnTckxHyAZfDA/uiaTnUcClzYO5dxX54Y/yU8sJg71fX4Wru3TqsqvE61Ilguxi1LK5wo40tVyj5wowRq/Ppf9eYFNJP4Funm/g4ISj/XwX2WdCES0yXpV1emJafjg5j64sHOzoOcUBtmp79r3fkVOfjH25hahZYOa3uNum4aos2koToxfN/OPcdNzw9G1ZX1c2asVzmrXSHvc4ufcqmHNwIPkQwT410/JsRhbJP6XFdsqfn5RaVJv5+kpV5i8umK9pq+iaOK489PQm/p0fGxqwK5nQ1+vWKrkwtd/9n8KAGDroRNRXzxE683Zm3AgrwgX/9M6hmBW7s7FXr1PwDw6za55aEwEcWYu6NMsZsNa/S0P6xI4YzlapzWtU+XXqIxT9OU3yD47Dxeg21Mz0G7MlESHEtSzP64LGLoZKyVl5Xj06+BrQlnNazFsO2TV1BLb0vbxb9cE7QiOxFHTaCK7Jq0yEcSJMXRPBFjx5IVY8aT1EkUe7U8AABWLSURBVNVdTqkXcOzhCztW6kpg9iPn4c8XdsQ3956NZvXSo3+BGKhbI3Zt4aEEnZTlAP+YGZtdu+z0Y4Srt8ab1QhiqwllVVGZvqhg5m2yZ9FNJoI4eeHKrrhlQFuc2yEDDWqloUEt601rzmrXCEv+5rtyaZ30FKx7ZpjPsZv6tw37nqdm1MGDQzqgV5uGIYdX1q9pXVjfatpbobLq1YxPN1RWDHZUI+cpsxgC/M7cbIszK2f9vnwsiHLPhlCKg2xUVVVMBHHStF4NPD2yK1LCTJISETStVwM//WWwz/EaqW5vgb3m6YvRtnEtAEDvNpENNw2VCB64wHqS21Om3dYqq0565ROBefp/jVQX2mfUrnI8RGZWw0fTKjmR0UqstxmtV8OeCysmgiSV2SSw0Mt6fCiyHh+KOukp3oLdqp/BSqiZj+0zagddLbKqaqVV/hfX/Ee0eMxQnH9601iEFLU+fnM96OThsZhLEssr+Fir7Gz7cJgIqpFUtwtN9C0ym+udsMO7tsALV3bDoA5NAAAPD+1g+dxQNYJere0r6KJZLyiU9FSXratsXtwl+BC/S7u3CPoYVW/Bhm8mK7sWOGYiSGLj7+yH6Q+fa/nYpd1b4JPbz8JN/dvixn5tcMNZbQAAl/c4xfL8Xm20wv75K7v6HP/H73qgYW3r/opYqJkaPhE09JtcZSQ7sxqpbkxdY1+H48AOwTc8Cjf7NFJ/PK99TF6HYseuzle7RLteVMSva8urUkycc1oTnN48cHMLQOtLOK9jhnf9mBHdWmDbiyPQPsN6mOhfLuqIaQ8Pwu/7he9kjqVaEdQI/DurR/bUkplRyzHY1VEGhN5BLFbtvMa2mESVZVelmIngJGKMMf7h/oH49r5zfB5LcbvQqbk2NPW13/WwHKZ6usWOSpd0axFyZqS/16/t4XO/RgQ1gnZ+/SHGEg7Gkt1Gh7NVWf2IvktcJELN5g11nWVOVG9c1zPi9/MXaX8OAPzuzPjPPCbnYiI4CXVrVR89Qyxed82ZrTDqXK2ZorMpIXxz39lY9vhQn3PH/r43Pri5D2qnudEpSO3E7KrerbBw9AXo2lJ7XXMfwY392lg+x78fwZhGX66AifecjVl/Pg+AdcfzA0Os+0T83Xp2Jl65pgea17Oe4Ga1h4Dh6jNbobHefBZJDSeYaGoW1/VtjR8fGFjp94rEv37f29bXp9iza/Y4E4FDjezZEmuevhhntKhIBLXSUtBYb583+hwMq5+6GFMfqlj868tR/TH+zn6Wr31Kg5reDmhzH8ETl3TGgv87P+B8/5EQRjNoebnCmW0bejvG61Zh6Nyfhmo1h2ArmoYa3ZTqdmHZExdi+0uXID2CGk4w0SQRBaBrS/s2MbpzYDsM69IcT1za2bb3qIwB7RvHdEE+igwTgYMFG+O/+fnheP4K305ll0t8prf3a98Y55zWBOv9JroZjOGq5uacmmlutGpYK+Bc/20mjULZ/wr6rkGV62y9d/CpqK8XLuHWnzdPonvnxl6Y9+hgn8cHndYEY2/sHdWchjS3Cy9e1Q3Xn2VdI7IS6YWfeUGyaNRMc8PlEtwxsJ1tI1Eq485B7fD13WdbPlaVC4GThV0j55gIKECq2xV0E5O2jWuhY7OKDulgw0ONMf/GaKXQ71fxXttfugTpelu6/9X7ZT1OwRkt6uG9P5zpc/whU/NQll/TFqBttGIIthyxsclMZuOKRNWoVhraNvYt8F0uwSXdW+DT288K+ZnM6tVMwQ1ntfH5nLESbnjuoxefbnnc3HdjRPX4JWcAAN41NRllNq6FM0PMo+jcIrCvqSpcIpb7CJ/XMQOjKnkhEG9PXWZfLSuandCiwURAUZn15/Mw2W99+J8fPR/PjuyCifcM8B4bckYzbHxuGLq2rI8vR/XHh7f0CXitCXf1R530FNzit5SFURD4X727XYKpDw3CsK7NcZGpA/tBUyJoUifdO+va0OWUiiYWo5bx3zv6+RSSxiY1bnMzVYhy26pmY+XBIR3wxaj+ACo/9C/UbNJwa1D5j7wymJupjLhuOKsN5j06GMO6NseTepNR8/o1MPGes/Hy1d0wvGvg4ocPDonx1qtivdRysacMfTIbhXzqm9dXdOT/8dzgSePfN/fBnQPbhXytIZ3CT168/3zrzx6uqA6WnBOJiYCikup2BbTpt2lcCzcNyMSZbX3/UNP14ZL92jfGkDMqCu7/3NoXf7+sMwac2jignwKoKJhCteK8f9OZ2PrCCAAIuII0zxbt3qo+ru/b2nv/mZFd0aROGvq3b4T7zj8NUx8ahB8fGIg/ntsenZrXxQhTYRdJwR2s89nw5ws74rSmWid7NGvJG52CK5+8CAvHDAl6XqgY3S4JaHYzNDLNHTFewiWCto1rQ0S8AwNEz4bX9W2Dd/1qYgBwcZfm+M9tfSvdROXPJeKz8Y+hqLQcA05tjM3PDw/63JE9K7Z9DdYxXyvNjaGdm+GCMAV9emr4ovGewadaHg83JuC+IAnE4BLfSYzmC5thXe2Z3MhGN4q78y3+CFc+eZH3Ctwo10NtVSkiQdu2jeafyQ8ORMdmdX36Nq7o1RJXmPaJNiehaX6T9/xrFv42PDsMIsDpj08LeV5FzIHH6qSn4MZ+bXBd39YoLCnz7sVrfPL6Fh2nT1/eBa/N2Ij8Io/3szWolYqXruqOIydK8LdvtF3a/ntHPzSuYz1ZcJBpAp1W2CvL+MI1RYgIzj+9Ka7odQrGzq36fhAusV4E0ViTP9IlFoL97kTaxn5aRh0M6tAEpzeri49+2YYZfzoP14/71Wdf5dpB+tiMJH5Gi3pYvy/P57H7zteSx5BOTbEl5zi2H9b2Q2hSJx0/PHAO6tdM9faRHchbiKXbj+LFq7phS84JLMw+hD8EGXlXVUwElBTMBV4dvSmkdiWHap6aUQf7covQon7NSq3Nsu3FESj2lIedAxHJHAkzc0LKfn44vlq2G1f3buWdX2DeED5UgXVd39ZYufsYJi3f402aV/RsiWF6bearZbvQrnFtDDi1MQBg3qOD8cq0jd6NYb6592yfGoFVE5gxUquvX3PMD/cPxPs/bwlYVvqRC0/HvYNPw4G8Ilzwj3kAtMURu1psjdmsXjoOmLat/OH+gXh28jos2XYEHZvVtRxddTzE1pljhncK2Os7XIEfbl3/GmlufHaHNirusUvOgIhEnESM86ya7fq3134mH97aFwBw6Hgx+jw3C2Xl5WhR37pWleJy4ab+bSNacbiymAgo6VzeoyUO5hXjpgGR/+L/945+aKrvuTD2xt5YsfuYb2EXBRGJupCPxjVntkKK2xUwRNdczvTNDN5Bm57iwoMXdED2weMYekYzrN2b59MJ/s29vpMJ2zaujVeu6e5NBP4d+BPvPhvfrtjj7aQHgPYZdTD7kfOQ6ddZ3q1VfbxyTfeAROByCWqnp/jMbK+TnoJHLuyIpTuOYliX5vhdn1bezuAXp67H+/O2olaaG91a1cf//jjA5/W+ufdsXPmvhd77I021OH9/PK+iiWbuXwYjv6gUP4TZUjRc04+5Gc9IGvVrpuLwiZJgT/EyalG92zREvRqp6HJKPfx7gfW2q0bSO/s0676ceGEioKTjdonPH3ckBpo6RevXSsV5HYOvHRRrN/Zrg8/1jdib1UvH3y/rgiXbjmCdX7MAoDUnBaulGMtcDOrQJGC58i9H9cd14xYB0AqmzCa18f39A/HZr9sBhJ+sVjs9BdMfPtfyartbq/ro1ipwzsKpQZYrCbeibKpbvJ3v4Sb8PXCB9eO92jTEsC7NMW3tfqx75mLL5Tm+vnsAfsn23YfZmKU+fW3oPZF7hZhwCQC9LUZKfXL7WZix7gCe/XFdwGPv/r437hm/HEBFH0GtNDcmjOqPw8eLQySCFMx55DycYtHHcmbbRli6/SiaBGneiyUmAqIqeuHKbhjSqSnu+CQLY4afgRHdWmBEN+tOvVA1jRqpbvz4wMCAJTcArcO9beNa2HHYd4/dTnofRyTDdIOtWxVrSx8bGvG6UKH6IN64vicOnygJSDxX9W6JFbuOoU9mo6AjiZrW9e3Eb16vBvbnFXnfz9w0dGpGbTwzsisKS8pwatM6aFQ7zbKfonWjWrj17EzLRDDc9PP21s70t2hcJx1z/zIYL05ZH9DUBiDk+mBX924Z9PFYYiIgioEhZzTDzD+diw4W6zVFI9Rs4m/vPQf79A3NDX0zG2HB/50fs1E7kerXrlFAs5Eh2O57ZsZopFDt7jVS3Zaf6/Vrw6/39Pt+bXC0oARfLNmFJy7tjMGnZ6CLRX8FAMx+ZHDY1zMY7f7GqKpFphFdPz96PmqkubByl7ZtqnmORbsmtTHu5sAh1KGkuF1V/n2K+L3i8i5EDmD3H23D2mmWS4ZHOqchlr70a9OPljEQwK6+mBS3Cw8P7YiH9aVFjH2DqzozV0Qw4a7+3tqV0akOaMOoAeDCzjUw/eFz41YDiwUmAiKKu7v0CV92joQxM1qC/PtIKlOTMkZjhVKdkgDAREBECVAj1R3xyrGxer8xwzv5TGzc9NzwpFpnKZGYCIjIEfxHokWzP8TJjt8EEZHD2ZoIRGSYiGwUkWwRGW3xeLqIfKk/vlhEMu2Mh4iIAtmWCETEDWAsgOEAOgO4QUT812e9A8BRpdRpAP4J4GW74iEiImt21gjOApCtlNqqlCoB8AWAkX7njATwiX77awBDJNwiIEREFFN2JoKWAHaZ7u/Wj1meo5TyAMgFEDA2S0RGiUiWiGTl5OTYFC4RkTNVi85ipdQ4pVQfpVSfjIz4rSFDROQEdiaCPQBam+630o9ZniMiKQDqAzgMIiKKGzsTwVIAHUSknYikAbgewPd+53wP4Bb99jUA5ihl1/bMRERkRewsd0VkBIA3ALgBfKSUel5EngGQpZT6XkRqAPgMQC8ARwBcr5TaGuY1cwDsqGRITQAcquRz44UxVl2yxwckf4zJHh+Q/DEmW3xtlVKWbeu2JoJkIyJZSqnolgCMM8ZYdckeH5D8MSZ7fEDyx5js8ZlVi85iIiKyDxMBEZHDOS0RjEt0ABFgjFWX7PEByR9jsscHJH+MyR6fl6P6CIiIKJDTagREROSHiYCIyOEckwjCLYkdpxhai8hcEVknImtF5CH9+FMiskdEVuj/RpieM0aPeaOIXBynOLeLyGo9liz9WCMRmSkim/X/G+rHRUTe0mNcJSK9bY7tdNP3tEJE8kTk4UR/hyLykYgcFJE1pmNRf2cicot+/mYRucXqvWIc46siskGP4xsRaaAfzxSRQtP3+Z7pOWfqvx/Z+ueIyUKRQeKL+udq5996kBi/NMW3XURW6Mfj/h1WmlLqpP8HbULbFgDtAaQBWAmgcwLiaAGgt367LoBN0JbofgrAXyzO76zHmg6gnf4Z3HGIczuAJn7HXgEwWr89GsDL+u0RAKYCEAD9ASyO8891P4C2if4OAZwLoDeANZX9zgA0ArBV/7+hfruhzTFeBCBFv/2yKcZM83l+r7NEj1v0zzHcxvii+rna/bduFaPf4/8A8GSivsPK/nNKjSCSJbFtp5Tap5Rart/OB7AegSuymo0E8IVSqlgptQ1ANrTPkgjmJcM/AXCF6finSrMIQAMRaRGnmIYA2KKUCjXTPC7foVLqZ2iz4/3fO5rv7GIAM5VSR5RSRwHMBDDMzhiVUjOUtvIvACyCtiZYUHqc9ZRSi5RWon1q+lwxjy+EYD9XW//WQ8WoX9VfC2BCqNew8zusLKckgkiWxI4r0XZj6wVgsX7ofr16/pHRhIDExa0AzBCRZSIySj/WTCm1T7+9H4CxC3giv9vr4ftHl0zfIRD9d5bo39PboV2dGtqJyG8iMk9EBunHWupxGeIRYzQ/10R+h4MAHFBKbTYdS5bvMCSnJIKkIiJ1AEwE8LBSKg/AuwBOBdATwD5o1ctEGqiU6g1td7n7RORc84P6VUxCxx2LtpDh5QC+0g8l23foIxm+s1BE5DEAHgDj9UP7ALRRSvUC8GcAn4tIvQSEltQ/Vz83wPfCJFm+w7CckggiWRI7LkQkFVoSGK+UmgQASqkDSqkypVQ5gA9Q0XSRkLiVUnv0/w8C+EaP54DR5KP/fzCRMUJLUsuVUgf0WJPqO9RF+50lJFYRuRXApQB+rycs6E0uh/Xby6C1u3fU4zE3H9kaYyV+ron6DlMAXAXgS+NYsnyHkXBKIohkSWzb6W2IHwJYr5R63XTc3KZ+JQBjRML3AK4XkXQRaQegA7ROJjtjrC0idY3b0DoT18B3yfBbAHxnivFmfSRMfwC5puYQO/lcfSXTd2gS7Xc2HcBFItJQbwK5SD9mGxEZBuCvAC5XShWYjmeItu84RKQ9tO9tqx5nnoj013+fbzZ9Ljvii/bnmqi/9aEANiilvE0+yfIdRiSRPdXx/AdtpMYmaFn5sQTFMBBa88AqACv0fyOgLcW9Wj/+PYAWpuc8pse8EXEYWQBttMVK/d9a47uCtoXobACbAcwC0Eg/LgDG6jGuBtAnDjHWhraBUX3TsYR+h9CS0j4ApdDafO+ozHcGrZ0+W/93WxxizIbWpm78Pr6nn3u1/vNfAWA5gMtMr9MHWoG8BcA70FcosCm+qH+udv6tW8WoH/8YwN1+58b9O6zsPy4xQUTkcE5pGiIioiCYCIiIHI6JgIjI4ZgIiIgcjomAiMjhmAjIsURkof5/pojcGOPX/pvVexElIw4fJccTkcHQVri8NIrnpKiKxdqsHj+ulKoTi/iI7MYaATmWiBzXb74EYJC+ZvyfRMQt2jr9S/XFzv6onz9YROaLyPcA1unHvtUX51trLNAnIi8BqKm/3njze+mziV8VkTX6evTXmV77JxH5WrT9Acbrs06JbJeS6ACIksBomGoEeoGeq5TqKyLpAH4RkRn6ub0BdFXa0scAcLtS6oiI1ASwVEQmKqVGi8j9SqmeFu91FbQF1HoAaKI/52f9sV4AugDYC+AXAOcAWBD7j0vkizUCokAXQVsLaAW0ZcIbQ1snBgCWmJIAADwoIiuhreXf2nReMAMBTFDaQmoHAMwD0Nf02ruVtsDaCmgbmxDZjjUCokAC4AGllM+Cb3pfwgm/+0MBDFBKFYjITwBqVOF9i023y8C/T4oT1giIgHxoW4capgO4R18yHCLSUV+J1V99AEf1JNAJ2taDhlLj+X7mA7hO74fIgLb1YbxWQyWyxCsOIm1lyzK9iedjAG9Ca5ZZrnfY5sB6K8FpAO4WkfXQVsBcZHpsHIBVIrJcKfV70/FvAAyAtrqrAvBXpdR+PZEQJQSHjxIRORybhoiIHI6JgIjI4ZgIiIgcjomAiMjhmAiIiByOiYCIyOGYCIiIHO7/Ad+s1P5nJtvmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder!.plotLosses()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Progress bar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's nice to keep track of where we're at in the training with a progress bar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "import Foundation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "func formatTime(_ t: Float) -> String {\n",
    "    let t = Int(t)\n",
    "    let (h,m,s) = (t/3600, (t/60)%60, t%60)\n",
    "    return h != 0 ? String(format: \"%02d:%02d:%02d\", h, m, s) : String(format: \"%02d:%02d\", m, s)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"01:18\"\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "formatTime(78.23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "public struct ProgressBar{\n",
    "    let total: Int\n",
    "    let length: Int = 50\n",
    "    let showEvery: Float = 0.2\n",
    "    let fillChar: Character = \"X\"\n",
    "    public var comment: String = \"\"\n",
    "    private var waitFor: Int = 0\n",
    "    private var startTime: UInt64 = 0\n",
    "    private var lastPrint: UInt64 = 0\n",
    "    private var lastShow: UInt64 = 0\n",
    "    private var estimatedTotal: Float = 0.0\n",
    "    private var bar: String = \"\"\n",
    "    \n",
    "    public init(_ c: Int) { total = c }\n",
    "    \n",
    "    public mutating func update(_ val: Int){\n",
    "        lastShow = DispatchTime.now().uptimeNanoseconds\n",
    "        if val == 0 { startTime = lastShow } \n",
    "        else {\n",
    "            let averageTime = Float(lastShow - startTime) / (1e9 * Float(val))\n",
    "            estimatedTotal = Float(total) * averageTime\n",
    "        }\n",
    "        if val == 0 || lastShow - lastPrint >= Int(1e9 * showEvery) { update_bar(val) }\n",
    "    }\n",
    "    \n",
    "    public mutating func update_bar(_ val: Int){\n",
    "        lastPrint = lastShow\n",
    "        let prevLength = bar.count\n",
    "        bar = String(repeating: fillChar, count: (val * length) / total)\n",
    "        bar += String(repeating: \"-\", count: length - (val * length) / total)\n",
    "        let pct = String(format: \"%.2f\", 100.0 * Float(val)/Float(total))\n",
    "        let elapsedTime = Float(lastShow - startTime) / 1e9\n",
    "        let remaingTime = estimatedTotal - elapsedTime\n",
    "        bar += \" \\(pct)% [\\(val)/\\(total) \\(formatTime(elapsedTime))<\\(formatTime(remaingTime))\"\n",
    "        bar += comment.isEmpty ? \"]\" : \" \\(comment)]\"\n",
    "        if bar.count < prevLength { bar += String(repeating: \" \", count: prevLength-bar.count) }\n",
    "        print(bar, terminator:\"\\r\")\n",
    "        fflush(stdout)\n",
    "    }\n",
    "    \n",
    "    public func remove(){\n",
    "        print(String(repeating: \" \", count: bar.count), terminator:\"\\r\")\n",
    "        fflush(stdout)\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                              \r"
     ]
    }
   ],
   "source": [
    "var tst = ProgressBar(100)\n",
    "for i in 0...100{\n",
    "    tst.update(i)\n",
    "    usleep(50000)\n",
    "}\n",
    "tst.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "extension Learner {\n",
    "    public class ShowProgress: Delegate {\n",
    "        var pbar: ProgressBar? = nil\n",
    "        var iter: Int = 0\n",
    "        \n",
    "        public override func epochWillStart(learner: Learner) {\n",
    "            pbar = ProgressBar(learner.data.train.count)\n",
    "        }\n",
    "        \n",
    "        public override func validationWillStart(learner: Learner) {\n",
    "            if pbar != nil { pbar!.remove() }\n",
    "            pbar = ProgressBar(learner.data.valid.count)\n",
    "        }\n",
    "        \n",
    "        public override func epochDidFinish(learner: Learner) {\n",
    "            if pbar != nil { pbar!.remove() }\n",
    "        }\n",
    "        \n",
    "        public override func batchWillStart(learner: Learner) {\n",
    "            if learner.currentIter == 0 {pbar!.update(0)}\n",
    "        }\n",
    "        \n",
    "        public override func batchDidFinish(learner: Learner) {\n",
    "            pbar!.update(learner.currentIter)\n",
    "        }\n",
    "        \n",
    "        public override func trainingDidFinish(learner: Learner) {\n",
    "            if pbar != nil { pbar!.remove() }\n",
    "        }\n",
    "    }\n",
    "    \n",
    "    public func makeShowProgress() -> ShowProgress { return ShowProgress() }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeShowProgress(), \n",
    "                     learner.makeAvgMetric(metrics: [accuracy]), learner.makeRecorder(),\n",
    "                     learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: [0.29616934, 0.9166]                                                  \n",
      "Epoch 1: [0.24793093, 0.9303]                                                  \n",
      "                                                                           \r"
     ]
    }
   ],
   "source": [
    "learner.fit(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Annealing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "/// A non-generalized learning rate scheduler\n",
    "extension Learner where Opt.Scalar: BinaryFloatingPoint {\n",
    "    public class LRScheduler: Delegate {\n",
    "        public override var order: Int { return 1 }\n",
    "        public typealias ScheduleFunc = (Float) -> Float\n",
    "\n",
    "        // A learning rate schedule from step to float.\n",
    "        public var scheduler: ScheduleFunc\n",
    "        \n",
    "        public init(scheduler: @escaping (Float) -> Float) {\n",
    "            self.scheduler = scheduler\n",
    "        }\n",
    "        \n",
    "        override public func batchWillStart(learner: Learner) {\n",
    "            learner.opt.learningRate = Opt.Scalar(scheduler(learner.pctEpochs/Float(learner.epochCount)))\n",
    "        }\n",
    "    }\n",
    "    \n",
    "    public func makeLRScheduler(scheduler: @escaping (Float) -> Float) -> LRScheduler {\n",
    "        return LRScheduler(scheduler: scheduler)\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "public func linearSchedule(start: Float, end: Float, pct: Float) -> Float {\n",
    "    return start + pct * (end - start)\n",
    "}\n",
    "\n",
    "public func makeAnnealer(start: Float, end: Float, schedule: @escaping (Float, Float, Float) -> Float) -> (Float) -> Float { \n",
    "    return { pct in return schedule(start, end, pct) }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.037\n"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "let annealer = makeAnnealer(start: 1e-2, end: 0.1, schedule: linearSchedule)\n",
    "annealer(0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n",
    "let recorder = learner.makeRecorder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeShowProgress(), \n",
    "                     learner.makeAvgMetric(metrics: [accuracy]), recorder,\n",
    "                     learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std),\n",
    "                     learner.makeLRScheduler(scheduler: annealer)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: [0.24022955, 0.9273]                                                  \n",
      "Epoch 1: [0.14354073, 0.9569]                                                  \n",
      "                                                                             \r"
     ]
    }
   ],
   "source": [
    "learner.fit(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhUhdn+8e9D2Pd9J4R9R4QA4r6golURoRW1LtWKtvXtW/trIW4Vd9Ra61ttfbFq1bq1BDSuuO8bQSEhgUAIW9gh7BCyPb8/Zng7pQOyZHImmftzXbkyc85J5s7JcuecM3li7o6IiMj+agUdQERE4pMKQkREolJBiIhIVCoIERGJSgUhIiJR1Q46QGVp3bq1p6SkBB1DRKRamTt37iZ3bxNtXY0piJSUFDIzM4OOISJSrZjZigOt0ykmERGJSgUhIiJRqSBERCQqFYSIiESlghARkahiWhBmNsbM8sws38zSoqw/2cy+NbMyM5uw37orzWxJ+OXKWOYUEZH/FLOCMLMk4DHgHKA/cImZ9d9vs5XAVcAL+71tS+B2YCQwArjdzFrEKquIiPynWB5BjADy3b3A3UuAl4CxkRu4+3J3zwIq9nvbs4F33b3I3bcA7wJjYphVRKTacXdenrOS93LXx+T9x7IgOgGrIu4XhpdV2tua2SQzyzSzzI0bNx5xUBGR6mbl5t1c9tevmZKezSvzVsfkMar1X1K7+3RgOkBqaqr+85GI1HjlFc7fvljO72fnkVTLuGfcQC4ZnhyTx4plQawGukTc7xxedqhve+p+b/tRpaQSEammFq/fweQZWcxbtZXT+7blnnED6dCsQcweL5YFMQfoZWbdCP3AnwhceohvOxu4N+LC9FnATZUfUUQk/pWUVfCXj5by6IdLaFK/Do9MHMIFx3TEzGL6uDErCHcvM7MbCP2wTwKecvccM7sTyHT3DDMbDswCWgDnm9kd7j7A3YvM7C5CJQNwp7sXxSqriEi8mr9qK1PSs1i0bgcXHNOR28/vT6vG9arksc29Zpy6T01NdU1zFZGaYk9JOQ+/t5i/flpA2yb1ufvCgYzu367SH8fM5rp7arR11foitYhITfTl0s3cNDOL5Zt3c8mIZG46ty9N69ep8hwqCBGROLG9uJRpby3iha9X0rVVQ164diTH92gdWB4VhIhIHHh/4XpumbWADTuKmXRyd24c3ZsGdZMCzaSCEBEJ0Oade7njtVwy5q+hT7smPH75MIZ0aR50LEAFISISCHcnY/4a7ngtlx3Fpdw4ujc/O7UHdWvHz5BtFYSISBVbu20Pt85awPuLNnBMl+Y8MH4wfdo3CTrWf1BBiIhUkYoK56U5q7jvzYWUVlRw6w/68ZMTupFUK7Z/8HakVBAiIlVg+aZdpM3M4quCIo7v0YppFw0muVXDoGMdlApCRCSGysoreOrzZTz0zmLqJtVi2kWDuHh4l5iPyagMKggRkRhZtG47U2ZkMb9wG6P7tePuCwfSvln9oGMdMhWEiEgl21tWzmMfLuXPH+bTrEEd/nTJsZw3uEO1OGqIpIIQEalE363cwpT0LBav38m4Yztx23n9admobtCxjogKQkSkEuwuKeOhdxbz1OfLaN+0Pk9fNZzT+rYNOtZRUUGIiBylL/I3kTYzm5VFu/nxcclMGdOXJgEM16tsKggRkSO0bU8p9725kJfmrKJb60a8POk4RnZvFXSsSqOCEBE5Au/krOPWVxawaederjslNFyvfp1gh+tVNhWEiMhh2LRzL1Mzcng9ay192zfhr1emMrhzfAzXq2wqCBGRQ+DuvDJvNXe8lsvuveX8vzN7c/2pPaiTFD/D9SqbCkJE5Hus2bqHW2Zl82HeRoYmN+f+8YPp1S7+hutVNhWEiMgBVFQ4z3+zkmlvLqTC4fbz+3PFqJS4Ha5X2VQQIiJRFGzcSVp6Nt8sL+LEnq2576JBdGkZ38P1KpsKQkQkQll5BX/9bBkPv7uYerVr8cCEwfxwWOdqNyajMqggRETCctdsZ3L6fBas3s7ZA9px19iBtG1afYbrVTYVhIgkvL1l5Tz6QT5/+WgpzRvW4c+XDeWcge0T8qghkgpCRBLa3BVFTEnPJn/DTsYP7cxt5/WjecPqOVyvsqkgRCQh7dpbxoOz83jmy+V0bNaAZ64ewSm92wQdK66oIEQk4Xy6ZCM3zcymcMserhzVld+O6UvjevpxuD/tERFJGNt2l3L3G7n8c24h3ds04p/Xj2J4SsugY8UtFYSIJIS3F6zjtlcXULSrhJ+f2oNfntGrxg3Xq2wqCBGp0TbsKGZqRg5vZq+jf4emPH3VcAZ2ahZ0rGpBBSEiNZK7k/7tau56PZc9peX89uw+TDq5e40erlfZVBAiUuMUbtnNzbMW8MnijaR2bcG08YPp2bZx0LGqHRWEiNQYFRXOc1+t4P63FwFwxwUDuPy4rtRKkOF6lU0FISI1wtKNO5kyI4vMFVs4uXcb7h03kM4tEmu4XmVTQYhItVZaXsH0Twp45P0lNKiTxO9/eAzjh3ZK+DEZlSGmV2vMbIyZ5ZlZvpmlRVlfz8xeDq//2sxSwsvrmNkzZpZtZgvN7KZY5hSR6mnB6m2MffRzHpydx+h+bXn31yczIUEnr8ZCzI4gzCwJeAw4EygE5phZhrvnRmx2DbDF3Xua2UTgfuBi4IdAPXcfZGYNgVwze9Hdl8cqr4hUH8Wl5Tzy/hKmf1JAy0Z1efzHQxkzsEPQsWqcWJ5iGgHku3sBgJm9BIwFIgtiLDA1fHsG8KiFqt+BRmZWG2gAlADbY5hVRKqJOcuLmDIji4JNu/jhsM7c+oP+NGtYJ+hYNVIsC6ITsCrifiEw8kDbuHuZmW0DWhEqi7HAWqAhcKO7F+3/AGY2CZgEkJycXNn5RSSO7NxbxgNvL+LZL1fQuUUDnrtmBCf10nC9WIrXi9QjgHKgI9AC+NTM3tt3NLKPu08HpgOkpqZ6lacUkSrx8eKN3DwzmzXb9nDV8Sn89uw+NNJwvZiL5R5eDXSJuN85vCzaNoXh00nNgM3ApcDb7l4KbDCzz4FUoAARSRhbdpVw1xu5zPx2NT3aNGLG9aMY1lXD9apKLJ/FNAfoZWbdzKwuMBHI2G+bDODK8O0JwAfu7sBK4HQAM2sEHAcsimFWEYkj7s6b2Ws58+GPyZi3hv86vSdv/PIklUMVi9kRRPiawg3AbCAJeMrdc8zsTiDT3TOAJ4HnzCwfKCJUIhB69tPTZpYDGPC0u2fFKquIxI8N24u57dUFzM5Zz6BOzXj26pH079g06FgJyUK/sFd/qampnpmZGXQMETlC7s4/5xZy9+u57C2r4MYze/PTE7tRW8P1YsrM5rp7arR1usojIoFbVbSbm2Zm81n+JkaktGTa+EF0b6PhekFTQYhIYMornGe/XM4Db+dRy+CuCwdy2YhkDdeLEyoIEQnEkvU7mJKexbcrt3JqnzbcM24QnZo3CDqWRFBBiEiVKi2v4PGPlvKnD/JpVC+JP148hLFDOmp+UhxSQYhIlcku3MZvZ8xn0bodnDe4A1MvGEDrxvWCjiUHoIIQkZgrLi3n4fcW88QnBbRuXI/plw/jrAHtg44l30MFISIx9XXBZtJmZrNs0y4mDu/CTef2o1kDDderDlQQIhITO4pLuf/tRfz9q5V0admA5386khN6tg46lhwGFYSIVLoPF23g5lnZrN9ezE9P7Mavz+pNw7r6cVPd6DMmIpWmaFcJd76Wwyvz1tCrbWP+/LPjOTa5RdCx5AipIETkqLk7r2etZWpGDtv2lPLfZ/Ti56f1oF7tpKCjyVFQQYjIUVm/vZhbZi3gvYXrGdy5Gc9fO5K+7TVcryZQQYjIEXF3Xp6zinveXEhJWQW3nNuPn5yQouF6NYgKQkQO24rNu7hpZjZfLN3MyG4tuX/8YFJaNwo6llQyFYSIHLLyCufpz5fx+3fyqFOrFveOG8TE4V00XK+GUkGIyCHJW7eDyelZzF+1lTP6tuXucQPp0EzD9WoyFYSIHFRJWQV//iifxz7Mp0n9OjwycQgXHKPheolABSEiBzR/1VYmz8gib/0Oxg7pyO/O608rDddLGCoIEfkPe0rK+cO7eTz52TLaNqnPX69IZXT/dkHHkiqmghCRf/PF0k3cNDObFZt3c+nIZNLO6UvT+hqul4hUECICwPbiUu57cxEvfrOSrq0a8uK1xzGqR6ugY0mAVBAiwnu567nllWw27tjLpJO7c+Po3jSoqzEZiU4FIZLANu/cyx2v5ZIxfw192zdh+uWpHNOledCxJE6oIEQSkLuTMX8NUzNy2Lm3jBtH9+Znp/agbm2NyZB/UUGIJJi12/Zw66wFvL9oA0O6NOeBCYPp3a5J0LEkDqkgRBJERYXz4pyV3PfmIsornNvO689Vx6eQpDEZcgAqCJEEsGzTLtLSs/h6WREn9GzFfeMGk9yqYdCxJM6pIERqsLLyCp76fBkPvbOYurVrcf/4QfwotYvGZMghUUGI1FAL125nSnoWWYXbOLN/O+6+cCDtmtYPOpZUIyoIkRpmb1k5j324lD9/mE+zBnV49NJj+cGgDjpqkMOmghCpQb5duYUpM7JYsmEn447txO/O60+LRnWDjiXVlApCpAbYXVLG72cv5ukvltGhaX2evmo4p/VtG3QsqeZUECLV3Of5m0ibmcWqoj1cflxXJo/pQxMN15NKoIIQqaa27Snl3jcW8nLmKrq1bsTLk45jZHcN15PKo4IQqYbeyVnHra8sYPOuEq4/pQe/Gt2L+nU0XE8qV0wLwszGAI8AScBf3X3afuvrAc8Cw4DNwMXuvjy8bjDwv0BToAIY7u7FscwrEu827tjL1NdyeCNrLf06NOXJK4czqHOzoGNJDRWzgjCzJOAx4EygEJhjZhnunhux2TXAFnfvaWYTgfuBi82sNvB34HJ3n29mrYDSWGUViXfuzqzvVnPn67ns3lvOb87qzXWn9KBOkobrSezE8ghiBJDv7gUAZvYSMBaILIixwNTw7RnAoxZ6svZZQJa7zwdw980xzCkS11Zv3cMts7L5KG8jQ5NDw/V6ttVwPYm9WBZEJ2BVxP1CYOSBtnH3MjPbBrQCegNuZrOBNsBL7v7A/g9gZpOASQDJycmV/gGIBKmiwnn+6xVMe2sRDkw9vz+Xj9JwPak68XqRujZwIjAc2A28b2Zz3f39yI3cfTowHSA1NdWrPKVIjBRs3ElaejbfLC/ipF6tuXfcILq01HA9qVqxLIjVQJeI+53Dy6JtUxi+7tCM0MXqQuATd98EYGZvAkOB9xGpwcrKK3ji02U8/N5i6teuxYMTBjNhWGeNyZBAxLIg5gC9zKwboSKYCFy63zYZwJXAl8AE4AN333dqabKZNQRKgFOAh2OYVSRwOWu2MSU9iwWrt3P2gHbcNXYgbTVcTwIUs4IIX1O4AZhN6GmuT7l7jpndCWS6ewbwJPCcmeUDRYRKBHffYmZ/IFQyDrzp7m/EKqtIkIpLy/nTB0t4/OMCWjSsy18uG8o5gzoEHUsEc68Zp+5TU1M9MzMz6Bgih2XuiiImz8hi6cZdjB/amdvO60fzhhquJ1UnfH03Ndq6eL1ILVKj7dpbxoOz83jmy+V0bNaAZ64ewSm92wQdS+TfqCBEqtgnizdy08xs1mzbwxXHdeW3Y/rSuJ6+FSX+6KtSpIps213KXW/kMmNuId3bNOIf141ieErLoGOJHJAKQqQKvL1gLbe9mkPRrhJ+fmoPfnmGhutJ/PveggjPVMpx975VkEekRtmwo5jbX83hrQXrGNCxKU9fNZyBnTRcT6qH7y0Idy83szwzS3b3lVURSqS6c3dmzC3k7jcWsqe0nMlj+nDtSd01XE+qlUM9xdQCyDGzb4Bd+xa6+wUxSSVSja0q2s3Ns7L5dMkmhqe0YNr4wfRo0zjoWCKH7VAL4raYphCpASoqnGe/XM4Ds/Mw4M6xA/jxyK7U0nA9qaYOqSDc/eNYBxGpzvI37CQtPYvMFVs4uXcb7h03kM4tNFxPqreDFoSZ7SA06uI/VgHu7k1jkkqkmigtr2D6JwU88t4SGtRN4qEfHsNFQztpuJ7UCActCHfXfyUROYAFq7cxeUYWuWu384NBHZh6wQDaNKkXdCyRSqO/gxA5TMWl5Tzy/hKmf1JAy0Z1efzHwxgzsH3QsUQqnQpC5DDMWV7ElBlZFGzaxY9SO3PLuf1p1rBO0LFEYkIFIXIIdu4t44G3F/Hslyvo3KIBf79mJCf2ah10LJGYUkGIfI8P8zZwy8xs1m4v5icnpPCbs/rQSMP1JAHoq1zkALbsKuGu13OZ+d1qerZtzIzrj2dY1xZBxxKpMioIkf24O29mr+P2jAVs3V3KL0/vyS9O70m92hquJ4lFBSESYcP2Ym59ZQHv5K5nUKdmPHv1SPp31J/7SGJSQYgQOmr4Z2Yhd72RS0lZBTed05drTuxGbQ3XkwSmgpCEt3JzaLjeZ/mbGNGtJdMuGkR3DdcTUUFI4iqvcP72xXJ+PzuPpFrG3RcO5NIRyRquJxKmgpCEtGT9DianZ/Hdyq2c1qcN94wbRMfmDYKOJRJXVBCSUErKKnj846U8+kE+jeol8ceLhzB2SEcN1xOJQgUhCSOrcCuTZ2SxaN0Ozj+mI7ef35/WjTVcT+RAVBBS4xWXlvPwu4t54tMC2jSpxxNXpHJm/3ZBxxKJeyoIqdG+KthMWnoWyzfv5pIRXUg7px/NGmi4nsihUEFIjbSjuJRpby3i+a9XktyyIS/8dCTH99RwPZHDoYKQGueDReu5ZdYC1m8v5qcnduPXZ/WmYV19qYscLn3XSI1RtKuEO1/L4ZV5a+jdrjF/vux4jk3WcD2RI6WCkGrP3Xktay1TM3LYUVzKf5/Ri1+c1pO6tTUmQ+RoqCCkWlu3LTRc772F6zmmczPunzCSvu01XE+kMqggpFpyd16as4p731hIaUUFt5zbj6tP7EaSxmSIVBoVhFQ7KzbvIi09my8LNnNc95ZMu2gwKa0bBR1LpMZRQUi1UV7hPP35Mn7/Th51atXivosGcXFqFw3XE4kRFYRUC3nrQsP15q/ayuh+bbn7wkG0b1Y/6FgiNVpMn+ZhZmPMLM/M8s0sLcr6emb2cnj912aWst/6ZDPbaWa/iWVOiV8lZRX88b3FnPenT1lVtJv/ueRYnrgiVeUgUgVidgRhZknAY8CZQCEwx8wy3D03YrNrgC3u3tPMJgL3AxdHrP8D8FasMkp8m7dqK1NmZJG3fgdjh3Tk9vMH0LJR3aBjiSSMWJ5iGgHku3sBgJm9BIwFIgtiLDA1fHsG8KiZmbu7mV0ILAN2xTCjxKE9JeU89E4eT32+jLZN6vPklamc0U/D9USqWiwLohOwKuJ+ITDyQNu4e5mZbQNamVkxMIXQ0ccBTy+Z2SRgEkBycnLlJZfAfLF0E2np2aws2s2lI5NJO6cvTetruJ5IEOL1IvVU4GF333mwf+Ti7tOB6QCpqaleNdEkFrYXl3Lfmwt58ZtVpLRqyIvXHseoHq2CjiWS0GJZEKuBLhH3O4eXRdum0MxqA82AzYSONCaY2QNAc6DCzIrd/dEY5pWAvJe7nlteyWbjjr1cd3J3fjW6Nw3qJgUdSyThxbIg5gC9zKwboSKYCFy63zYZwJXAl8AE4AN3d+CkfRuY2VRgp8qh5tm8cy9TX8vltflr6Nu+CU9ckcrgzs2DjiUiYTEriPA1hRuA2UAS8JS755jZnUCmu2cATwLPmVk+UESoRKSGc3denbeGO17LYefeMn59Zm+uP6WHhuuJxBkL/cJe/aWmpnpmZmbQMeR7rNm6h1tfWcAHizYwpEtzHpgwmN7tmgQdSyRhmdlcd0+Nti5eL1JLDVNR4bzwzUqmvbWI8grntvP6c9XxKRquJxLHVBASc8s27SItPYuvlxVxQs9W3DduMMmtGgYdS0S+hwpCYqasvIInP1vGH95dTN3atXhg/GB+mNqZgz11WUTihwpCYiJ3zXampGeRvXobZ/Zvx90XDqRdU81PEqlOVBBSqfaWlfPoB/n85aOlNG9Yh8cuHcq5g9rrqEGkGlJBSKWZu2ILU9KzyN+wk4uO7cRt5/WnhYbriVRbKgg5artLynhwdh5/+2I5HZrW5+mfDOe0Pm2DjiUiR0kFIUflsyWbSJuZReGWPVwxqiuTx/SlcT19WYnUBPpOliOybU8p97yRyz8yC+nWuhH/uG4UI7q1DDqWiFQiFYQcttk567jtlQVs3lXCz07twX+f0Yv6dTRcT6SmUUHIIdu4Yy9TM3J4I3st/To05ckrhzOoc7OgY4lIjKgg5Hu5OzO/Xc2dr+eyp6Sc357dh0knd6dOkobridRkKgg5qNVb93DzzGw+XryRYV1bcP/4QfRsq+F6IolABSFRVVQ4f/96Bfe/tQgHpp7fnytGpVBLw/VEEoYKQv7D0o07SUvPYs7yLZzUqzX3jhtEl5YarieSaFQQ8n9Kyyt44tMC/vjeEurXrsWDEwYzYZiG64kkKhWEALBg9TampGeRs2Y7Ywa0584LB9C2iYbriSQyFUSCKy4t508fLOHxjwto0bAuf7lsKOcM6hB0LBGJAyqIBJa5vIjJ6VkUbNzFhGGdufUH/WjeUMP1RCREBZGAdu0NDdd75svldGzWgGevHsHJvdsEHUtE4owKIsF8vHgjN8/MZs22PVw5KoXfnt2HRhquJyJR6CdDgti6u4S7Xl9I+reFdG/TiH9eN4rUFA3XE5EDU0EkgLey13Lbqzls2V3CL07rwX+druF6IvL9VBA12Ibtxfzu1RzezlnHgI5Neebq4QzoqOF6InJoVBA1kLszY24hd72eS3FZBVPG9OXak7pRW8P1ROQwqCBqmFVFu7l5VjafLtnE8JQWTBs/mB5tGgcdS0SqIRVEDVFe4Tz75XIenJ2HAXeNHcBlI7tquJ6IHDEVRA2Qv2EHU9KzmbtiC6f0bsM94wbSuYWG64nI0VFBVGOl5RX878dL+Z/382lYL4k//OgYxh3bScP1RKRSqCCqqQWrt/HbGVksXLudHwzuwNTzB9CmSb2gY4lIDaKCqGaKS8v543tLeOLTAlo2qsv/Xj6Mswe0DzqWiNRAKohq5JtlRaSlZ1GwaRcXp3bh5nP70axhnaBjiUgNpYKoBnYUl/LA23k899UKOrdowN+vGcmJvVoHHUtEajgVRJz7MG8Dt8zMZu32Yq4+oRu/Obs3Devq0yYisaefNHFqy64S7no9l5nfraZn28bMuP54hnVtEXQsEUkgMZ29YGZjzCzPzPLNLC3K+npm9nJ4/ddmlhJefqaZzTWz7PDr02OZM564O69nrWH0Hz4mY/4afnl6T9745YkqBxGpcjE7gjCzJOAx4EygEJhjZhnunhux2TXAFnfvaWYTgfuBi4FNwPnuvsbMBgKzgU6xyhov1m8v5rZXFvBO7noGdWrG3386kn4dmgYdS0QSVCxPMY0A8t29AMDMXgLGApEFMRaYGr49A3jUzMzdv4vYJgdoYGb13H1vDPMGxt35R+Yq7n5jISVlFdx0Tl+uOVHD9UQkWLEsiE7Aqoj7hcDIA23j7mVmtg1oRegIYp/xwLfRysHMJgGTAJKTkysveRVauXk3aTOz+GLpZkZ0a8n94wfTrXWjoGOJiMT3RWozG0DotNNZ0da7+3RgOkBqaqpXYbSjVl7h/O2L5fx+dh5JtYy7LxzIpSOSNVxPROJGLAtiNdAl4n7n8LJo2xSaWW2gGbAZwMw6A7OAK9x9aQxzVrnF63cweUYW81Zt5bQ+bbhn3CA6Nm8QdCwRkX8Ty4KYA/Qys26EimAicOl+22QAVwJfAhOAD9zdzaw58AaQ5u6fxzBjlSopq+Dxj5fypw+W0LhebR6ZOIQLjumo4XoiEpdiVhDhawo3EHoGUhLwlLvnmNmdQKa7ZwBPAs+ZWT5QRKhEAG4AegK/M7PfhZed5e4bYpU31uav2sqU9CwWrdvB+cd0ZOr5/WnVWMP1RCR+mXu1OnV/QKmpqZ6ZmRl0jP+wp6Sch99bzF8/LaBNk3rcfeEgzuzfLuhYIiIAmNlcd0+Nti6uL1JXd18u3cxNM7NYvnk3l4zowk3n9qNpfQ3XE5HqQQURA9uLS5n21iJe+HolyS0b8sJPR3J8Tw3XE5HqRQVRyT5YtJ6bZy5gw45irj2pG78+sw8N6iYFHUtE5LCpICrJ5p17ufP1XF6dt4Y+7Zrw+OXDGNKledCxRESOmAriKLk7GfPXcMdruewoLuVXo3vx81N7Ure2xmSISPWmgjgKa7ft4dZZC3h/0QaO6dKcB8YPpk/7JkHHEhGpFCqII1BR4bw0ZxX3vbmQ0ooKbv1BP35yQjeSNCZDRGoQFcRhWr5pF2kzs/iqoIhR3VsxbfwgurbScD0RqXlUEIeovMJ56rNlPPRuHnVq1eK+iwYxcXgXjckQkRpLBXEIFq3bzpQZWcwv3Mbofm25+8JBtG9WP+hYIiIxpYI4iL1l5Tz24VL+/GE+zRrU4U+XHMt5gzvoqEFEEoIK4gC+W7mFKelZLF6/kwuHdOR35w+gZaO6QccSEakyKoj97C4p46F3FvPU58to37Q+T12Vyul9NVxPRBKPCiLCF/mbSJuZzcqi3Vw2Mpm0c/rSRMP1RCRBqSCAbXtKue/Nhbw0ZxUprRry0qTjOK57q6BjiYgEKuELIqtwK9c+m8nGHXu57pTu3Di6N/XraLieiEjCF0Ryy4b0bteEJ65IZXBnDdcTEdkn4QuiecO6PHfNyKBjiIjEHY0cFRGRqFQQIiISlQpCRESiUkGIiEhUKggREYlKBSEiIlGpIEREJCoVhIiIRGXuHnSGSmFmG4EVR/EuWgObKilOLMR7Poj/jPGeD+I/Y7znA2U8XF3dvU20FTWmII6WmWW6e2rQOQ4k3vNB/GeM93wQ/xnjPR8oY2XSKSYREYlKBSEiIlGpIP5letABvke854P4zxjv+SD+M8Z7PlDGSqNrECIiEpWOIEREJCoVhIiIRJXwBWFmY8wsz8zyzSwtoAxdzOxDM8s1sxwz++/w8qlmttrM5oVfzo14m5vCmfPM7OwqyrnczLLDWTLDy1qa2baaePQAAAZrSURBVLtmtiT8ukV4uZnZ/4QzZpnZ0CrI1ydiX80zs+1m9qsg96OZPWVmG8xsQcSyw95nZnZlePslZnZlFWR80MwWhXPMMrPm4eUpZrYnYl8+HvE2w8JfH/nhj8NinPGwP6+x+n4/QL6XI7ItN7N54eWB7MMj4u4J+wIkAUuB7kBdYD7QP4AcHYCh4dtNgMVAf2Aq8Jso2/cPZ60HdAt/DElVkHM50Hq/ZQ8AaeHbacD94dvnAm8BBhwHfB3A53Yd0DXI/QicDAwFFhzpPgNaAgXh1y3Ct1vEOONZQO3w7fsjMqZEbrff+/kmnNvCH8c5Mc54WJ/XWH6/R8u33/qHgN8FuQ+P5CXRjyBGAPnuXuDuJcBLwNiqDuHua9392/DtHcBCoNNB3mQs8JK773X3ZUA+oY8lCGOBZ8K3nwEujFj+rId8BTQ3sw5VmOsMYKm7H+yv62O+H939E6AoyuMezj47G3jX3YvcfQvwLjAmlhnd/R13Lwvf/QrofLD3Ec7Z1N2/8tBPumcjPq6YZDyIA31eY/b9frB84aOAHwEvHux9xHofHolEL4hOwKqI+4Uc/AdzzJlZCnAs8HV40Q3hw/yn9p2KILjcDrxjZnPNbFJ4WTt3Xxu+vQ5oF3DGfSby79+Q8bQfD3efBb0vryb02+w+3czsOzP72MxOCi/rFM61T1VlPJzPa1D78SRgvbsviVgWT/vwgBK9IOKKmTUG0oFfuft24C9AD2AIsJbQYWqQTnT3ocA5wC/M7OTIleHfegJ/3rSZ1QUuAP4ZXhRv+/H/xMs+OxAzuwUoA54PL1oLJLv7scCvgRfMrGlA8eL287qfS/j3X1biaR8eVKIXxGqgS8T9zuFlVc7M6hAqh+fdfSaAu69393J3rwCe4F+nPwLJ7e6rw683ALPCedbvO3UUfr0hyIxh5wDfuvv6cN642o8c/j4LJKeZXQWcB1wWLjLCp202h2/PJXROv3c4T+RpqJhnPILPa5XvRzOrDVwEvByRO2724fdJ9IKYA/Qys27h3zonAhlVHSJ8jvJJYKG7/yFieeQ5+3HAvmdIZAATzayemXUDehG6uBXLjI3MrMm+24QuYi4IZ9n3rJorgVcjMl4RfmbOccC2iNMqsfZvv7HF036MeNzD2WezgbPMrEX4NMpZ4WUxY2ZjgMnABe6+O2J5GzNLCt/uTmifFYRzbjez48Jfz1dEfFyxyni4n9cgvt9HA4vc/f9OHcXTPvxeQV4hj4cXQs8cWUyoxW8JKMOJhE4zZAHzwi/nAs8B2eHlGUCHiLe5JZw5jyp4pgOhZ37MD7/k7NtXQCvgfWAJ8B7QMrzcgMfCGbOB1Cral42AzUCziGWB7UdCRbUWKCV0TvmaI9lnhK4D5IdfflIFGfMJna/f9/X4eHjb8eHP/zzgW+D8iPeTSuiH9FLgUcKTGmKY8bA/r7H6fo+WL7z8b8D1+20byD48kheN2hARkagS/RSTiIgcgApCRESiUkGIiEhUKggREYlKBSEiIlGpIESiMLMvwq9TzOzSSn7fN0d7LJF4o6e5ihyEmZ1KaGLoeYfxNrX9X4Puoq3f6e6NKyOfSCzpCEIkCjPbGb45DTgpPLf/RjNLstD/SpgTHhJ3XXj7U83sUzPLAHLDy14JDzbM2Tfc0MymAQ3C7+/5yMcK/wX1g2a2IPw/AS6OeN8fmdkMC/2PhufDf2krElO1gw4gEufSiDiCCP+g3+buw82sHvC5mb0T3nYoMNBDI6YBrnb3IjNrAMwxs3R3TzOzG9x9SJTHuojQ4LljgNbht/kkvO5YYACwBvgcOAH4rPI/XJF/0RGEyOE5i9C8pHmERrK3IjRLB+CbiHIA+KWZzSf0/xS6RGx3ICcCL3poAN164GNgeMT7LvTQYLp5hP7pjEhM6QhC5PAY8F/u/m/D8sLXKnbtd380MMrdd5vZR0D9o3jcvRG3y9H3rlQBHUGIHNwOQv8Gdp/ZwM/C49kxs97h6bb7awZsCZdDX0L/RnKf0n1vv59PgYvD1znaEPo3llUxXVYkKv0WInJwWUB5+FTR34BHCJ3e+TZ8oXgj0f8t5NvA9Wa2kNBE0a8i1k0HsszsW3e/LGL5LGAUoYm5Dkx293XhghGpcnqaq4iIRKVTTCIiEpUKQkREolJBiIhIVCoIERGJSgUhIiJRqSBERCQqFYSIiET1/wGnybUtd1POQgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "recorder.plotLRs()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "More annealing functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "public func constantSchedule(start: Float, end: Float, pct: Float) -> Float {\n",
    "    return start\n",
    "}\n",
    "\n",
    "public func cosineSchedule(start: Float, end: Float, pct: Float) -> Float {\n",
    "    return start + (1 + cos(Float.pi*(1-pct))) * (end-start) / 2\n",
    "}\n",
    "\n",
    "public func expSchedule(start: Float, end: Float, pct: Float) -> Float {\n",
    "    return start * pow(end / start, pct)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "// export\n",
    "public func combineSchedules(pcts: [Float], schedules: [(Float) -> Float]) -> ((Float) -> Float){\n",
    "    var cumPcts: [Float] = [0]\n",
    "    for pct in pcts {cumPcts.append(cumPcts.last! + pct)}\n",
    "    func inner(pct: Float) -> Float{\n",
    "        if (pct == 0.0) { return schedules[0](0.0) }\n",
    "        if (pct > 1.0)  { return schedules.last!(1.0) }\n",
    "        let i = cumPcts.firstIndex(where: {$0 >= pct})! - 1\n",
    "        let actualPos = (pct-cumPcts[i]) / (cumPcts[i+1]-cumPcts[i])\n",
    "        return schedules[i](actualPos)\n",
    "    }\n",
    "    return inner\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "let mySchedule = combineSchedules(pcts: [0.3, 0.7], \n",
    "                                  schedules: [makeAnnealer(start: 0.3, end: 0.6, schedule: cosineSchedule),\n",
    "                                              makeAnnealer(start: 0.6, end: 0.2, schedule: cosineSchedule)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n",
    "let recorder = learner.makeRecorder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeShowProgress(), \n",
    "                     learner.makeAvgMetric(metrics: [accuracy]), recorder,\n",
    "                     learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std),\n",
    "                     learner.makeLRScheduler(scheduler: mySchedule)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: [0.2004789, 0.9431]                                                   \n",
      "Epoch 1: [0.14118196, 0.9605]                                                  \n",
      "                                                                           \r"
     ]
    }
   ],
   "source": [
    "learner.fit(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhU5dnH8e+dHUiAhIQ1AcK+QyCAiuBSZXEBFCugtloX1Erd64tvtQutVq3VVsWFutbKqlbpWxSpgiCKJIGwbwkQkrCFfQnZ7/ePOdgRhz2TMzO5P9c1FzPPOWfmlxOSO+c85zyPqCrGGGPM8cLcDmCMMSYwWYEwxhjjkxUIY4wxPlmBMMYY45MVCGOMMT5FuB2guiQmJmrr1q3djmGMMUElKytrt6om+VoWMgWidevWZGZmuh3DGGOCiojknWiZnWIyxhjjkxUIY4wxPlmBMMYY45MVCGOMMT5ZgTDGGOOTXwuEiAwVkfUikiMiE06wzvUiskZEVovIFK/2m0Vko/O42Z85jTHG/JDfLnMVkXBgEnA5UABkiMgsVV3jtU574FFggKruE5HGTnsC8BsgHVAgy9l2n7/yGmOM+T5/3gfRD8hR1U0AIjINGAGs8VrnDmDSsV/8qrrLaR8CzFXVvc62c4GhwFQ/5jV+UlZRxcrCA2zceYjdh0uprILICCE6IpxG9aJIqBdFYmw0LRvVJTY6ZG7NMSbo+fOnsQWQ7/W6AOh/3DodAERkERAO/FZVPz3Bti2O/wARGQeMA2jZsmW1BTfVI2PLXt5bnMec1Ts5Wl55Wts0rR9Dm6R6dG1en94t4+ndKp4m9WP8nNQY44vbf65FAO2Bi4FkYIGIdD/djVV1MjAZID093WY+ChBrtx/kydlrWbhxN3ExEYxMa8FFHRLp2rwBjetHEy5CRZVytKySvcVl7D1Sxq6DpWzZc4TcosPk7jrMO1/n8beFmwFo1agul3RszMUdkzivTSNiIsNd/gqNqR38WSAKgRSv18lOm7cC4FtVLQc2i8gGPAWjEE/R8N52vt+SmmpRXlnFK/NzeeHzjcTFRPDYlZ25sX8r6kT98Bd6RDjERIYTXy+Ktj5GgSmtqGTNtoMs3bqfRTm7mZaxlbe/3kLdqHCGdm3KNb1bcEHbRMLDpAa+MmNqJ/HXlKMiEgFsAH6E5xd+BnCDqq72WmcoMFZVbxaRRGAZ0AunYxro7ay6FOhzrE/Cl/T0dLWxmNyzv7iMn7+3lK9z9zCiV3N+N7wrDetGVdv7l5RX8s2mPXy6cgezV23nUEkFjeOiGd03hZvOa2WnoYw5SyKSparpPpf5c05qEbkC+Aue/oU3VfUJEZkIZKrqLBER4M94OqArgSdUdZqz7a3A/zpv9YSqvnWyz7IC4Z4tu49wy1tL2La/hCev7c51fZL9+nkl5ZV8sW4X72cVMG/9LsJFuKJ7M269MJVeKQ39+tnGhBrXCkRNsgLhjs27jzBm8jeUVVTxt5+mk946oUY/P2/PEd75Oo+ZmfkcKq3gog5J3H9Ze9JaxtdoDmOClRUI4xd5e45w/WvfUF6pTLmjP52a1ncty+HSCt79Jo/JC3LZV1zOxR2TeGRIJ7o0dy+TMcHACoSpdvuOlHHtK1+zv7iMaePOp2PTOLcjAXCktIK/f5PHawtyOXi0nNF9W/LQ4A4kxka7Hc2YgHSyAmFjMZkzVlJeybh3Myncf5S//TQ9YIoDQL3oCO6+uC1fPnwJt1yQyszMfC7503z+tmATFZVVbsczJqhYgTBn7DcfryZjyz7+/OOeNd7ncLoa1I3k11d3Yc4Dg+ibmsATs9cy8uVFrCo84HY0Y4KGFQhzRj7IKmB6Zj73XNKWq3s2dzvOKbVNiuWNm9N55cbe7DxYyohJi/jjJ2s5WnZ6d3YbU5tZgTCnbePOQzz20Sr6pybwwGUd3I5z2kSEYd2b8Z8HLuK63sm89uUmrnpxoR1NGHMKViDMaSmrqOIXU5dRLzqcF8emEREefP91GtSN5OnrevCP2/pzuLSCa15exOQFuVRVhcaFGsZUt+D7KTeuePGLjazbcYhnrutB4yC/a/nC9ol8et8gLu3UmCdnr+Mnb37LrkMlbscyJuBYgTCntKJgPy/Pz2VU72Qu7dTE7TjVIr5eFK/e1Ienru1OVt4+rnrhKzK3nHAkF2NqJSsQ5qRKKyr55cwVJMZG8euru7gdp1qJCGP6teSjewZQNyqcMZMX89aizYTKvUHGnCsrEOakXl+4mfU7D/HHa7vToE6k23H8olPT+nw8/kIu7pjE7/61hvunZ9tVTsZgBcKcROH+o7z0RQ5DujYJmVNLJ9KgTiSTf5LOw4M7MGv5Nsb8bbH1S5hazwqEOaE//N8aFOXxq0Lr1NKJhIUJ4y9tz6s39WHDjkNcM+lr1u845HYsY1xjBcL4tGBDEZ+s2sH4S9qRHF/X7Tg1akjXpsy483zKK6sY9crXfLmhyO1IxrjCCoT5gYrKKib+3xpaN6rLHYPauB3HFd2TG/Dx+AGkJNTl1rczmJGRf+qNjAkxViDMD8zMKiBn12EevaIz0RG1d/7nZg3q8P5d53NB20Y88sEKJi/IdTuSMTXKrwVCRIaKyHoRyRGRCT6W3yIiRSKS7Txu91pW6dU+y585zX8Vl1Xw3NwN9GkVz+Auod0xfTrqRUfwxs19ubJHM56cvY6nPllnl8GaWiPCX28sIuHAJOByoADIEJFZqrrmuFWnq+p4H29xVFV7+Suf8e2NhZspOlTKqzf1xjMjrImKCOOFMWk0qBPJq1/msr+4jCeu6U54mO0fE9r8ViCAfkCOqm4CEJFpwAjg+AJhAsTuw6W8tmATQ7o2oU+rwBzG2y3hYcITI7uRUDeKl+blcKSskuev7xmUY1IZc7r8+b+7BeDds1fgtB1vlIisEJH3RSTFqz1GRDJFZLGIjPT1ASIyzlkns6jIrjQ5Vy/Py+VoeSWPDO3kdpSAJCI8PKQjE4Z14l/Lt3Hf9GybhMiENH8eQZyOfwFTVbVURO4E3gEudZa1UtVCEWkDfCEiK1X1e72EqjoZmAyeKUdrMnio2XWwhPe+zePatBa0TYp1O05Au+uitoQJPDl7HSj8ZUwvIu1IwoQgfxaIQsD7iCDZafuOqu7xevk68IzXskLn300iMh9IA+wyEj959ctNVFQp4y9t53aUoDBuUFsE4YnZa1GUv45JsyJhQo4//0dnAO1FJFVEooAxwPeuRhKRZl4vhwNrnfZ4EYl2nicCA7C+C7/xPnpo1aie23GCxh2D2vDYlZ2ZvXIH905dZqebTMjx2xGEqlaIyHhgDhAOvKmqq0VkIpCpqrOAe0VkOFAB7AVucTbvDLwmIlV4ithTPq5+MtXEjh7O3u0DPTcS/uHfa3nkgxU8e11PwuzqJhMi/NoHoaqzgdnHtf3a6/mjwKM+tvsa6O7PbMZj1yHP0cM1dvRw1m4f2Ibiskqem7uBuOgIfju8q10ibEKC253UxmWTjx09XGJHD+fiF5e243BpBZMXbCI2JoJfDrErwUzwswJRix0oLmfKkq1c3aMZrRPt6OFciAiPDuvEoZIKJs3LJS4mkrsuaut2LGPOiRWIWuwf3+ZRXFbJnfaLrFqICH8Y2Y3DpRU89ck6GtSJZGy/lm7HMuasWYGopUrKK3lr0WYu6pBE52b13Y4TMsLDhOeu78mhknJ+9c+VNI6L5kedbUwrE5zswu1a6sOlhew+XMadF9XO4bz9KTI8jEk39KZr8waMn7KM7Pz9bkcy5qxYgaiFKquUyQty6ZHcgPPbNHI7TkiqFx3Bm7f0JSkumlvfzmDL7iNuRzLmjFmBqIU+W72DLXuKuXNQW7sc04+S4qJ5+2d9UVVufmsJuw+Xuh3JmDNiBaKWUVVeXbCJlgl1GdqtqdtxQl6bpFjeuKUvOw+WcNvbGRSXVbgdyZjTZgWillm6dR/L8/dzx8BUm8+ghvRuGc+LY3uzsvAAD05fTlWVjStpgoMViFrmrUVbiIuJYFSfZLej1CqXd2nCr67swqerd/DnuevdjmPMabHLXGuR7QeO8smqHdw6oDV1o+xbX9NuHdCanF2HmDQvl3aNY7kmzYq0CWx2BFGL/GNxHqrKT89v7XaUWklEmDiiG+e1SeB/3l9JVt5etyMZc1JWIGqJkvJKpny7lcs6NyEloa7bcWqtyPAwXr2pD80bxjDu71nk7y12O5IxJ2QFopaYlb2NfcXl3DKgtdtRar2GdaN445a+lFdWcfs7mRwutSubTGCyAlELqCpvfb2Fjk3i7Ma4ANE2KZaXb+xDTtFhHpiebVc2mYBkBaIWWLJ5L2u3H+RnA1rbjXEB5ML2iTx2ZWfmrtnJS/Ny3I5jzA/4tUCIyFARWS8iOSIywcfyW0SkSESyncftXstuFpGNzuNmf+YMdW8t2kLDupGM6NXC7SjmOLdc0Jpr01rw/H828PnanW7HMeZ7/FYgRCQcmAQMA7oAY0Wki49Vp6tqL+fxurNtAvAboD/QD/iNiMT7K2so27b/KJ+t2cGYvi2pExXudhxzHBHhyWu706VZfe6fls2mosNuRzLmO/48gugH5KjqJlUtA6YBI05z2yHAXFXdq6r7gLnAUD/lDGkzMvNR4Mb+Ni9BoIqJDOe1n/QhIly4890s67Q2AcOfBaIFkO/1usBpO94oEVkhIu+LSMqZbCsi40QkU0Qyi4qKqit3yKisUqZn5DOwfZJd2hrgkuPrMumG3uQWHebhGctRtU5r4z63O6n/BbRW1R54jhLeOZONVXWyqqaranpSUpJfAgazLzfsYvuBEm7ol3LqlY3rLmiXyP9e0ZlPV+/g5fm5bscxxq8FohDw/s2U7LR9R1X3qOqxMZBfB/qc7rbm1KZ8u5XEWJvRLJjcdmEqV/dszrOfrWfhRjsqNu7yZ4HIANqLSKqIRAFjgFneK4hIM6+Xw4G1zvM5wGARiXc6pwc7beY0bT9wlC/W7eL69GQiw90+UDSnS0R4elR32jeO5f5p2ew4UOJ2JFOL+e03h6pWAOPx/GJfC8xQ1dUiMlFEhjur3Ssiq0VkOXAvcIuz7V7g93iKTAYw0Wkzp2lGRgFVCmP6Wud0sKkbFcHLN/bmaHklv5i6lPLKKrcjmVpKQqUzLD09XTMzM92OERAqq5SBT39B28axvHtbf7fjmLP0cXYh903L5s5BbXj0is5uxzEhSkSyVDXd1zI79xCCFmwoYtuBEsb2s6OHYDaiVwtu7N+S1xZsYu4au4nO1DwrECFoypKtJMZGcZl1Tge9x6/qQrcW9XloRraN/GpqnBWIELPjQAlfrNvFdX1SiIqwb2+wi4kM5+Ub+qDAPVOWUlpR6XYkU4vYb5AQMzMzn8oqZUxfu/chVLRsVJdnf9yTFQUHeOLfa0+9gTHVxApECKmsUqZl5DOgXSNaJ9ZzO46pRkO6NuX2C1P5+zd5/HvFdrfjmFrCCkQIWbixiML9R61zOkT9z7BO9EppyIQPV1h/hKkRViBCyNQlW2lUL4rBXZq6HcX4QWR4GC+OTQOFe6cts/sjjN9ZgQgRuw6W8J+1u7iuT7J1ToewlIS6PHltd5Zt3c/zcze4HceEOPtNEiJmZhVQWaWMts7pkHd1z+aMTk/hlS9zWZSz2+04JoRZgQgBVVXK1CVbOb9NI9okxbodx9SA3wzvQpvEetw/PZvdh0tPvYExZ8EKRAj4Kmc3BfuOcoNNClRr1I2K4KUbenPgaDkPz1xOVVVoDJljAosViBAwdclWEupFMbir3Tldm3RuVp/HruzM/PVFvLlos9txTAiyAhHkdh0qYe6anVzXJ5noCJtzurb5yXmtGNylCU9/uo6VBQfcjmNCjBWIIDczs4AKu3O61hIRnrmuB4mx0fxi6lKbz9pUKysQQayqSpmWsZXz2iRY53Qt1rBuFH8dk8bWvcX8dtZqt+OYEOLXAiEiQ0VkvYjkiMiEk6w3SkRURNKd161F5KiIZDuPV/2ZM1gtyt1N/l67c9pAv9QE7rmkHe9nFTB7pQ3FYapHhL/eWETCgUnA5UABkCEis1R1zXHrxQH3Ad8e9xa5qtrLX/lCwdQlW4mvG8mQrnbntIF7f9SeBRuKePTDlfRuGU/TBjFuRzJBzp9HEP2AHFXdpKplwDRghI/1fg88Ddjku2eg6FApn63eyajeycREWue08QzF8fzoXpRVVPHQzGy79NWcM38WiBZAvtfrAqftOyLSG0hR1X/72D5VRJaJyJciMtDXB4jIOBHJFJHMoqKiagseDN7Pcjqn7fSS8dImKZZfX92FRTl77NJXc85c66QWkTDgOeAhH4u3Ay1VNQ14EJgiIvWPX0lVJ6tquqqmJyUl+TdwADnWOd0vNYF2ja1z2nzfmL4pXN6lCc98up412w66HccEMX8WiELA+9rLZKftmDigGzBfRLYA5wGzRCRdVUtVdQ+AqmYBuUAHP2YNKt9s2kPenmJusKMH44OI8PSoHjSoG8n905dRUm6z0Jmz488CkQG0F5FUEYkCxgCzji1U1QOqmqiqrVW1NbAYGK6qmSKS5HRyIyJtgPbAJj9mDSpTlmylQZ1IhnazzmnjW0K9KJ79cU827DzM05+uczuOCVJ+KxCqWgGMB+YAa4EZqrpaRCaKyPBTbD4IWCEi2cD7wF2qutdfWYPJ7sOlfLZ6h3VOm1O6qEMSt1zQmrcWbWHBhtrVR2eqh98ucwVQ1dnA7OPafn2CdS/2ev4B8IE/swWrD7IKKK9UxvazO6fNqU0Y1omvc3fz0MzlzLl/EAn1otyOZIKI3UkdRFQ9w3r3bR1P+yZxbscxQSAmMpy/jE7jQHE5Ez5Ygapd+mpOnxWIIPLNpj1s2VNsd06bM9KleX1+OaQjn63ZyfSM/FNvYIzDCkQQmbokn/oxEVzRvZnbUUyQue3CVAa0a8TE/1vD1j3FbscxQcIKRJDYc7iUOat2cK11TpuzEBYm/Om6noSHCQ/OyKbS7rI2p8EKRJB4P6uAssoqmzXOnLXmDeswcURXMvP28beFdtW4OTUrEEHg2JzT6a3i6WCd0+YcjOzVgmHdmvLcZxtYu93usjYnZwUiCBzrnLajB3OuRIQnrulO/TqRPDA9m9IKu8vanJgViCAw5VvPndPWOW2qQ0K9KJ4e1Z11Ow7x/NyNbscxAcwKRIArOlTKHLtz2lSzH3Vuwpi+Kby2IJeMLTZIgfHNCkSAm5mVT0WVckN/u3PaVK/HrupCcnwdHpqx3OayNj5ZgQhgVVXKtCX59E9NoF1j65w21Ss2OoI//7gX+fuKeeLfa92OYwKQFYgA9lXObrbutc5p4z/9UhMYN6gNU5ds5Yt1O92OYwKMFYgANuXbrSTUi7JhvY1fPXh5Bzo1jeOR91ey90iZ23FMALECEaB2HSxh7tqdXNcnmegI65w2/hMdEc5z1/fiwNEyHvtopQ3oZ75jBSJAzcjMp7JKbWA+UyO6NK/PA5d3YPbKHXycvc3tOCZAWIEIQJVVytQl+VzQthGpifXcjmNqiTsHtaVPq3ge/3gV2/YfdTuOCQB+LRAiMlRE1otIjohMOMl6o0RERSTdq+1RZ7v1IjLEnzkDzYKNRRTuP2qd06ZGhYcJz13fk8oq5ZfvL6fKBvSr9fxWIJw5pScBw4AuwFgR6eJjvTjgPuBbr7YueOaw7goMBV4+Nkd1bfDuN3kkxkYzuIt1Tpua1apRPR67sguLcvbw92+2uB3HuOyUBUJEwkXkbGY97wfkqOomVS0DpgEjfKz3e+BpoMSrbQQwTVVLVXUzkOO8X8jL23OEeet3cUP/lkRF2BlAU/PG9kvh4o5J/PGTdeTsOux2HOOiU/4GUtVKYL2InOn5jhaA9/RVBU7bd0SkN5Ciqv8+022d7ceJSKaIZBYVhcak7P9YnEe4CDfa6SXjEhHhmVE9qBMVzkMzsimvrHI7knHJ6f6JGg+sFpHPRWTWsce5fLCIhAHPAQ+d7Xuo6mRVTVfV9KSkpHOJExCOllUyPSOfId2a0qR+jNtxTC3WuH4MT4zszvKCA0yal+N2HOOSiNNc7/GzeO9CwHsAoWSn7Zg4oBswX0QAmgKzRGT4aWwbkj7KLuRgSQW3XNDa7SjGcGWPZny2pjkvfpHDJR0b0zOloduRTA07rSMIVf3S1+MUm2UA7UUkVUSi8HQ6f3fUoaoHVDVRVVuramtgMTBcVTOd9caISLSIpALtgSVn8fUFDVXlna+30LlZfdJbxbsdxxgAJg7vRuO4aB6Ykc3RMps7orY5aYEQkUMictDH45CInHQ6KlWtAMYDc4C1wAxVXS0iE52jhJNtuxqYAawBPgXucfpCQlbGln2s23GIm89vhXNEZYzrGtSN5Nkf92RT0RGe/vRsrlUxweykp5hU9ZyGEFXV2cDs49p+fYJ1Lz7u9RPAE+fy+cHknW+2UD8mghG9ftAXb4yrBrRL5GcDWvPWoi38qHNjBrYP/v4+c3rsOsoAsONACXNW7WB03xTqRNWa2z1MEPmfoZ1o1ziWh2cuZ3+xDehXW1iBCAD/WJxHpSo3ndfK7SjG+BQTGc7z1/diz+EyHv94tdtxTA2xAuGy4rIK/vFtHpd3bkKrRjbukglc3ZMbcN+P2vOv5dv4ODvkLyo0WIFw3QdZBewvLueOQW3cjmLMKd19cVvSWjbk8Y9Wsf2ADegX6qxAuKiySnnjq830TGlol7aaoBARHsbz1/eivFJ55P0VNqBfiLMC4aL/rN3Jlj3F3DEw1S5tNUGjdWI9HruqMws37rYB/UKcFQgXvb5wEy0a1mFoVxu11QSXG/q15BIb0C/kWYFwSXb+fjK27OPWC1OJCLdvgwkuIsLTo3pQNyqcB6bbgH6hyn4zueRvCzcRFxPB6L4pp17ZmADUuH4MT17TnZWFB3jxCxvQLxRZgXDBlt1H+GTldm7o35LY6NMdL9GYwDOsezOu7d2CSfNyWLZ1n9txTDWzAuGCV+bnEhkexm0XprodxZhz9tvhXWlaP4YHZyynuKzC7TimGlmBqGGF+4/ywdICxvRNoXGczflggl/9GM+Aflv2HOGPs21Av1BiBaKGTf4yFxEYd1Fbt6MYU23Ob9uI2wak8u7iPOav3+V2HFNNrEDUoF2HSpiakc+1acm0aFjH7TjGVKuHh3SkQ5NYHnl/BfuO2IB+ocAKRA16Y+FmKiqruPtiO3owoScmMpznR/diX3EZj320ClW7yzrYWYGoIbsPl/Lu4jyu6tGc1ok2KJ8JTV2bN+CByzvw75Xb+Th7m9txzDnya4EQkaEisl5EckRkgo/ld4nIShHJFpGvRKSL095aRI467dki8qo/c9aEV+bnUlJeyX2XtXc7ijF+deegtqS3iufxj1aRv7fY7TjmHPitQIhIODAJGAZ0AcYeKwBepqhqd1XtBTwDPOe1LFdVezmPu/yVsyZsP3CUdxfnMap3Mm2TYt2OY4xfhYcJz4/uBcAD07OpsLusg5Y/jyD6ATmquklVy4BpwAjvFVTVe17rekBInrR84fMcVNWOHkytkZJQl9+P7EZm3j4mzct1O445S/4sEC2AfK/XBU7b94jIPSKSi+cI4l6vRakiskxEvhSRgb4+QETGiUimiGQWFRVVZ/Zqs2X3EWZm5nNDv5Ykx9d1O44xNWZkWguuSWvBXz/fQFbeXrfjmLPgeie1qk5S1bbA/wCPOc3bgZaqmgY8CEwRkfo+tp2squmqmp6UFJgTqT83dwMR4cI9l7ZzO4oxNW7iiK60iK/DfdOyOVhS7nYcc4b8WSAKAe+R6JKdthOZBowEUNVSVd3jPM8CcoEOfsrpN0u37mPW8m3cfmEbu2va1EpxMZH8ZXQa2w+U8PhHq9yOY86QPwtEBtBeRFJFJAoYA8zyXkFEvE/KXwlsdNqTnE5uRKQN0B7Y5Mes1a6qSpn4rzUkxUXbfQ+mVuvTKp77ftSej7O38c9lBW7HMWfAb0OJqmqFiIwH5gDhwJuqulpEJgKZqjoLGC8ilwHlwD7gZmfzQcBEESkHqoC7VDWoTmLOWr6N7Pz9/Om6HtSzEVtNLXfPJe34auNuHv9oNX1aJtCykfXHBQMJlbsd09PTNTMz0+0YABwtq+SSZ+eTGBfFrHsuJCzMphM1pnD/UYb+ZQFtk2KZedf5RNpEWQFBRLJUNd3XMvsO+cFfP9/IjoMl/PqqrlYcjHG0aFiHJ6/pTnb+fl74fKPbccxpsAJRzdZuP8jfFm7iuj7J9EtNcDuOMQHl6p7Nua5PMpPm5bBkc1CdNa6VrEBUo8oq5dEPV9KgTiS/uqKz23GMCUi/Hd6VlIS63D9tGfuLbdTXQGYFohq9920e2fn7efyqzsTXi3I7jjEBKTY6ghfGpFF0uJSHZ66wUV8DmBWIarJ5t2c2rYHtExnZ6wc3jBtjvPRMaciEYZ35z9qdvLloi9txzAlYgagG5ZVV3D89m6iIMJ65rgci1jFtzKncOqA1l3VuwlOfrGV5/n634xgfrEBUgxe/yGF5/n6evKY7zRrYTHHGnA4R4dkf96BxXAz3TFnKgaM2FEegsQJxjr7auJuXvtjItWktuLJHM7fjGBNUGtaN4oWxaew4UMKED6w/ItBYgTgH+XuLGT91Ke0ax/L7kd3cjmNMUOrTKp5fDunIJ6t28O7iPLfjGC9WIM7SkdIK7nw3i8oqZfJP0m04DWPOwR0D23BJxyT+8H9rWVV4wO04xmEF4iyUV1Zx93tLWbfjIC+MSbM5po05R2Fhwp+v70VCvSjumbKUQzY0eECwAnGGKquUX85czoINRTx5TXcu6dTY7UjGhISEep7+iIJ9R5nw4UrrjwgAViDOQHllFQ9Mz+aj7G38ckhHxvRr6XYkY0JKv9QEHhrcgX+v2M5bdn+E66xAnKbDpRXc9W4Ws5ZvY8KwTtxzic0QZ4w/3DWoLZd1bsKTs9eSscXGa3KTFYjTkFt0mGsmLWLe+l38YWQ37rrIJgAyxl88/RE9SY6vwz3vLWXXoRK3I9Vafi0QIjJURNaLSI6ITPCx/C4RWSki2SLylYh08Vr2qLPdehEZ4s+cJ1JRWcXkBblc+cJC9hwp4x+39eem81q5EcWYWqVBndtuotMAABHuSURBVEhe/UkfDpaUM37KMsorq9yOVCv5rUA4U4ZOAoYBXYCx3gXAMUVVu6tqL+AZ4Dln2y54pijtCgwFXj42BWlNKCmv5MOlBQz+ywKenL2OC9slMfvegVzQLrGmIhhT63VqWp+nru3Bks17efqTdW7HqZX8efF+PyBHVTcBiMg0YASw5tgKqnrQa/16wLHLFkYA01S1FNgsIjnO+31T3SFLKyrJ3rqffcXlbNt/lGX5+5m/bheHSivo1DSO137Sh8Fdmtj4Ssa4YGRaC5Zt3cfrX22mV8uGXNWjuduRahV/FogWQL7X6wKg//Ericg9wINAFHCp17aLj9v2B0Okisg4YBxAy5Znd0XRoZIKRk/+70c1rR/D4K5NGZnWnAFtE21GOGNc9qsru7Cy8ACPvL+CTk3jaNc4zu1ItYbrt/+q6iRgkojcADwG3HwG204GJoNnTuqz+fz4ulG8d3t/GtSJpHFcNI3rx5zN2xhj/CQqIoyXb+zDVS8uZNzfs/jnPQNoUCfS7Vi1gj87qQuBFK/XyU7biUwDRp7ltmctPEwY0C6Rbi0aWHEwJkA1bRDDpBt6s3VvMfdOXUZlld1EVxP8WSAygPYikioiUXg6nWd5ryAi7b1eXgkcm8l8FjBGRKJFJBVoDyzxY1ZjTIDr36YRE0d048sNRTzzqXVa1wS/nWJS1QoRGQ/MAcKBN1V1tYhMBDJVdRYwXkQuA8qBfTinl5z1ZuDp0K4A7lHVSn9lNcYEhxv6t2TdjoO8tmATHZvGcW3vZLcjhTQJlfFO0tPTNTMz0+0Yxhg/K6+s4qdvLCFr6z6mjzuPtJbxbkcKaiKSparpvpbZndTGmKASGR7GpBt706R+NHe+m8WOA3antb9YgTDGBJ2EelG8/tO+zrwsmZSU2xlof7ACYYwJSh2bxvH86F6sKDzAQzOWU2VXNlU7KxDGmKA1uGtT/ndYZ/69cjtPz7Erm6qb6zfKGWPMubh9YCpb9xbz2pebSImvawNqViMrEMaYoCYi/ObqLhTuP8qvP15Fi4Z1bKbHamKnmIwxQS8iPIwXx6bRuVl97pmylFWFB9yOFBKsQBhjQkK96AjevKUvDetEcts7GWzbf9TtSEHPCoQxJmQ0qR/Dmz/rS3FpJTe/uYR9R8rcjhTUrEAYY0JKp6b1mfzTdPL2FvOztzM4UlrhdqSgZQXCGBNyzm/biJfGprGiYD93/SOL0gq7ke5sWIEwxoSkwV2b8tSoHizcuJsHpy+3IcLPgl3maowJWdenp3CguJwnZq+lQd1InhjZzaYPPgNWIIwxIe2OQW3YW1zGK/NziY2O4NFhnaxInCYrEMaYkPfIkI4cLqlg8oJNhIcJjwzpaEXiNFiBMMaEPBHhd8O7UqnKK/NzCRfhocEdrEicgl87qUVkqIisF5EcEZngY/mDIrJGRFaIyOci0sprWaWIZDuPWcdva4wxZyIsTPjDiG6M6ZvCS/Ny+OvnG0+9US3ntyMIEQkHJgGXAwVAhojMUtU1XqstA9JVtVhE7gaeAUY7y46qai9/5TPG1D5hYcKT13Snokr5y382EibCvT9q73asgOXPU0z9gBxV3QQgItOAEXjmmQZAVed5rb8YuMmPeYwxhrAw4elRPahS5bm5GzhaXml9EifgzwLRAsj3el0A9D/J+rcBn3i9jhGRTKACeEpVPzp+AxEZB4wDaNmy5TkHNsbUDuFhwp+u60lMZDivzM/lcEkFvxvelbAwKxLeAqKTWkRuAtKBi7yaW6lqoYi0Ab4QkZWqmuu9napOBiYDpKen210wxpjTFh4mPDGyG3HREby2YBNHSit45roeRITb/cPH+LNAFAIpXq+TnbbvEZHLgF8BF6lq6bF2VS10/t0kIvOBNCD3+O2NMeZsiQgThnUiLiaCZz/bwOHSCl4Ym0ZMZLjb0QKCP0tlBtBeRFJFJAoYA3zvaiQRSQNeA4ar6i6v9ngRiXaeJwID8Oq7MMaY6iIijL+0Pb+9ugufrdnJT99Ywv5iGwUW/FggVLUCGA/MAdYCM1R1tYhMFJHhzmp/AmKBmcddztoZyBSR5cA8PH0QViCMMX5zy4BUXhibRnb+fq595Wu27il2O5LrRDU0Tt2np6drZmam2zGMMUFuyea93PH3TCLDhddv7kuvlIZuR/IrEclS1XRfy6w3xhhjvPRLTeCDuy+gTlQ4YyZ/wycrt7sdyTVWIIwx5jjtGsfyz58PoHOz+tz93lKenbO+Vg4XbgXCGGN8SIyNZtq48xid7hma47Z3MjhwtNztWDXKCoQxxpxAdEQ4T43qzhPXdGNRzm5GvPQV63YcdDtWjbECYYwxJyEi3Ni/FVPvOI8jZZWMeGkR7y7OI1Qu8DkZKxDGGHMa0lsnMPvegfRv04jHP1rFne9mse9IaN8vYQXCGGNOU1JcNG/f0pfHruzMvPW7GPbXhSzK2e12LL+xAmGMMWcgLEy4fWAbPrx7AHWjwrnx9W959MMVHCwJvQ5sKxDGGHMWuic3YPZ9A7lzUBumZ+Qz+LkFfL52p9uxqpUVCGOMOUsxkeE8ekVn/vnzATSoE8lt72Ryx98zydtzxO1o1cIKhDHGnKOeKQ351y8u5JGhHVmUs5vLn1vAM5+u40hphdvRzokVCGOMqQZREWH8/OJ2zHv4Yq7q0YyX5+dy8bPzeWvRZkrKK92Od1asQBhjTDVqUj+G50b34sOfX0DbpHr87l9ruOhP83j3my2UVgRXobDRXI0xxo++zt3Nc59tIDNvH4mxUdx0Xitu7N+KpLhot6MBJx/N1QqEMcb4maryde4eXl+4iXnri4gKD+Oqns24rk8y56U2cnUu7JMViICYk9oYY0KZiDCgXSID2iWSW3SYtxdt4Z/LCvlwaSEtGtZhRK/mDOvWjK7N67taLI7n1yMIERkK/BUIB15X1aeOW/4gcDtQARQBt6pqnrPsZuAxZ9U/qOo7J/ssO4IwxgSTo2WVzF27kw+XFrBgQxFV6rlT+9KOjRnYIZE+reJp1qCO33O4copJRMKBDcDlQAGeOarHek8dKiKXAN+qarGI3A1crKqjRSQByATSAQWygD6quu9En2cFwhgTrPYcLmX++iK+WL+LBeuLOORcHtu8QQw9UxrSvnEsbRvH0iYxlsb1o0moF0VkePVcY+TWKaZ+QI6qbnJCTANGAN8VCFWd57X+YuAm5/kQYK6q7nW2nQsMBab6Ma8xxriiUWw0o/okM6pPMuWVVazdfpCsvH1k5e1jZeEB5qzewfHzFTWsG0m9qAiiIsLo1qIBL45Nq/Zc/iwQLYB8r9cFQP+TrH8b8MlJtm1x/AYiMg4YB9CyZctzyWqMMQEhMjyMHskN6ZHckJ8NSAWgpLySvD3FbN59hN2HS9l9uJQ9h8soLqukrLKKlHj/nIoKiE5qEbkJz+mki85kO1WdDEwGzykmP0QzxhjXxUSG07FpHB2bxtXo5/rzRrlCIMXrdbLT9j0ichnwK2C4qpaeybbGGGP8x58FIgNoLyKpIhIFjAFmea8gImnAa3iKwy6vRXOAwSISLyLxwGCnzRhjTA3x2ykmVa0QkfF4frGHA2+q6moRmQhkquos4E9ALDBTRAC2qupwVd0rIr/HU2QAJh7rsDbGGFMz7E5qY4ypxU52masN1meMMcYnKxDGGGN8sgJhjDHGJysQxhhjfAqZTmoRKQLyzuEtEoHd1RTHHwI9HwR+xkDPB5axOgR6PgisjK1UNcnXgpApEOdKRDJP1JMfCAI9HwR+xkDPB5axOgR6PgiOjGCnmIwxxpyAFQhjjDE+WYH4r8luBziFQM8HgZ8x0POBZawOgZ4PgiOj9UEYY4zxzY4gjDHG+GQFwhhjjE+1vkCIyFARWS8iOSIywcUcKSIyT0TWiMhqEbnPaf+tiBSKSLbzuMJrm0ed3OtFZEgNZNwiIiudHJlOW4KIzBWRjc6/8U67iMgLTr4VItK7BvJ19NpP2SJyUETud3sfisibIrJLRFZ5tZ3xfhORm531N4rIzX7O9ycRWedk+KeINHTaW4vIUa99+arXNn2c/x85ztcgfs54xt9Xf/28nyDfdK9sW0Qk22l3ZR+eFVWttQ88w5DnAm2AKGA50MWlLM2A3s7zOGAD0AX4LfCwj/W7OHmjgVTn6wj3c8YtQOJxbc8AE5znE4CnnedX4JlCVoDzgG9d+N7uAFq5vQ+BQUBvYNXZ7jcgAdjk/BvvPI/3Y77BQITz/GmvfK291zvufZY4mcX5Gob5eR+e0ffVnz/vvvIdt/zPwK/d3Idn86jtRxD9gBxV3aSqZcA0YIQbQVR1u6oudZ4fAtbiYx5uLyOAaapaqqqbgRw8X09NGwG84zx/Bxjp1f539VgMNBSRZjWY60dArqqe7O76GtmHqroAOH4+kzPdb0OAuaq6V1X3AXOBof7Kp6qfqWqF83IxnlkdT8jJWF9VF6vnN93fvb4mv2Q8iRN9X/32836yfM5RwPXA1JO9h7/34dmo7QWiBZDv9bqAk/9SrhEi0hpIA751msY7h/pvHjsVgTvZFfhMRLJEZJzT1kRVtzvPdwBNXMznbQzf/4EMlH14zJnuNzez3ornr9ljUkVkmYh8KSIDnbYWTqaazncm31e39uFAYKeqbvRqC6R9eEK1vUAEHBGJBT4A7lfVg8ArQFugF7Adz6GqWy5U1d7AMOAeERnkvdD5q8f166bFM8XtcGCm0xRI+/AHAmW/+SIivwIqgPecpu1AS1VNAx4EpohIfZfiBfT31ctYvv/HSiDtw5Oq7QWiEEjxep3stLlCRCLxFIf3VPVDAFXdqaqVqloF/I3/ngKp8eyqWuj8uwv4p5Nl57FTR86/x+YWd3PfDgOWqupOJ2/A7EMvZ7rfajyriNwCXAXc6BQxnNM2e5znWXjO6XdwsnifhqqJ/49n+n11Yx9GANcC071yB8w+PJXaXiAygPYikur81TkGmOVGEOc85RvAWlV9zqvd+7z9NcCxqyRmAWNEJFpEUoH2eDq4/JWvnojEHXuOpxNzlZPj2BU1NwMfe+X7qXNVznnAAa9TKv72vb/YAmUfHudM99scYLCIxDunUgY7bX4hIkOBR4Dhqlrs1Z4kIuHO8zZ49tkmJ+NBETnP+b/8U6+vyV8Zz/T76sbP+2XAOlX97tRRIO3DU3KzhzwQHniuGtmAp4r/ysUcF+I5zbACyHYeVwDvAiud9llAM69tfuXkXo+fr3bAc+XHcuex+ti+AhoBnwMbgf8ACU67AJOcfCuB9Braj/WAPUADrzZX9yGeYrUdKMdzXvm2s9lvePoCcpzHz/ycLwfP+fpj/xdfddYd5Xz/s4GlwNVe75OO55d0LvASzkgNfsx4xt9Xf/28+8rntL8N3HXcuq7sw7N52FAbxhhjfKrtp5iMMcacgBUIY4wxPlmBMMYY45MVCGOMMT5ZgTDGGOOTFQhjfBCRr51/W4vIDdX83v/r67OMCTR2masxJyEiF+MZMfSqM9gmQv870J2v5YdVNbY68hnjT3YEYYwPInLYefoUMNAZt/8BEQkXz1wJGc4gcXc6618sIgtFZBawxmn7yBnYcPWxwQ1F5CmgjvN+73l/lnP39J9EZJUzJ8Bor/eeLyLvi2eOhvecO22N8asItwMYE+Am4HUE4fyiP6CqfUUkGlgkIp856/YGuqlniGmAW1V1r4jUATJE5ANVnSAi41W1l4/PuhbPwHM9gURnmwXOsjSgK7ANWAQMAL6q/i/XmP+yIwhjzsxgPGMlZeMZjr0RnrF0AJZ4FQeAe0VkOZ75FFK81juRC4Gp6hmAbifwJdDX670L1DMwXTaeSWeM8Ss7gjDmzAjwC1X93kB5Tl/FkeNeXwacr6rFIjIfiDmHzy31el6J/eyaGmBHEMac3CE8U8AeMwe42xmaHRHp4Ixue7wGwD6nOHTCM43kMeXHtj/OQmC008+RhGcay5oaXdaYH7C/Qow5uRVApXOq6G3gr3hO7yx1OoqL8D0t5KfAXSKyFs+Ioou9lk0GVojIUlW90av9n8D5eEbMVeARVd3hFBhjapxd5mqMMcYnO8VkjDHGJysQxhhjfLICYYwxxicrEMYYY3yyAmGMMcYnKxDGGGN8sgJhjDHGp/8HnZ1J3HcFx1gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "recorder.plotLRs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "//Needs fixing \n",
    "//learner.recorder!.plotLRs()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "success\r\n"
     ]
    }
   ],
   "source": [
    "import NotebookExport\n",
    "let exporter = NotebookExport(Path.cwd/\"05_anneal.ipynb\")\n",
    "print(exporter.export(usingPrefix: \"FastaiNotebook_\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Swift",
   "language": "swift",
   "name": "swift"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
